rtl8812au dkms module not building for kernel 5.6 rc3 - here's a patch to fix it

[muncrief]

The rtl8812au dkms module fails to build for kernel 5.6 rc3. It also didn't build for rc1 and rc2, but I've included the failed build log for rc3 below.

rtl8812au-dkms-bad.log

[muncrief]

I created a patch to allow the rtl8812au dkms module to work with kernel 5.6 rc3, and probably future releases. The problem was that the proc_ops structure must be used instead of the file_operations structure in kernel 5.6. I found this information at https://github.com/openzfs/zfs/pull/9961 and derived the patch from one posted by satmandu at https://paste.ubuntu.com/p/wsS9GFHjyv.

Note that the member names of the procops structure are now prefixed with "proc", and that there's no "owner" member equivalent in proc_ops so it's removed when compiling for 5.6. Also note that I bumped the version to 5.6.4.3.

I tested the patched dkms module with kernels 5.4.19, 5.4.22, 5.5.6, and 5.6 rc3. And tested both USB 2 and 3, and 2.4 and 5GHz networks.

Someone will need to look at the tools and non-dkms installation, but this will at least get dkms users going. Note that you must uninstall all existing dkms modules with the unpatched code before applying the patch because of the version bump. Then patch the code and install and uninstall as usual.

----------- BEGIN PATCH ------------

diff --git a/dkms-install.sh b/dkms-install.sh
index 07c476b..139dd38 100755
--- a/dkms-install.sh
+++ b/dkms-install.sh
@@ -9,7 +9,7 @@ fi

 DRV_DIR="$(pwd)"
 DRV_NAME=rtl8812au
-DRV_VERSION=5.6.4.2
+DRV_VERSION=5.6.4.3

 cp -r $(pwd) /usr/src/${DRV_NAME}-${DRV_VERSION}

diff --git a/dkms-remove.sh b/dkms-remove.sh
index 6a8d6d6..e274670 100755
--- a/dkms-remove.sh
+++ b/dkms-remove.sh
@@ -9,7 +9,7 @@ fi

 DRV_DIR="$(pwd)"
 DRV_NAME=rtl8812au
-DRV_VERSION=5.6.4.2
+DRV_VERSION=5.6.4.3

 dkms remove ${DRV_NAME}/${DRV_VERSION} --all
 rm -rf /usr/src/${DRV_NAME}-${DRV_VERSION}
diff --git a/os_dep/linux/rtw_proc.c b/os_dep/linux/rtw_proc.c
index dbc60a4..316dc38 100644
--- a/os_dep/linux/rtw_proc.c
+++ b/os_dep/linux/rtw_proc.c
@@ -62,7 +62,7 @@ inline struct proc_dir_entry *rtw_proc_create_dir(const char *name, struct proc_
 }

 inline struct proc_dir_entry *rtw_proc_create_entry(const char *name, struct proc_dir_entry *parent,
-   const struct file_operations *fops, void * data)
+   const rtw_proc_op_t *fops, void * data)
 {
    struct proc_dir_entry *entry;

@@ -238,22 +238,38 @@ static ssize_t rtw_drv_proc_write(struct file *file, const char __user *buffer,
    return -EROFS;
 }

-static const struct file_operations rtw_drv_proc_seq_fops = {
+static const rtw_proc_op_t rtw_drv_proc_seq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_drv_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = seq_release,
+   .proc_write = rtw_drv_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_drv_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = seq_release,
    .write = rtw_drv_proc_write,
+#endif
 };

-static const struct file_operations rtw_drv_proc_sseq_fops = {
+static const rtw_proc_op_t rtw_drv_proc_sseq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_drv_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = single_release,
+   .proc_write = rtw_drv_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_drv_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = single_release,
    .write = rtw_drv_proc_write,
+#endif
 };

 int rtw_drv_proc_init(void)
@@ -4162,22 +4178,38 @@ static ssize_t rtw_adapter_proc_write(struct file *file, const char __user *buff
    return -EROFS;
 }

-static const struct file_operations rtw_adapter_proc_seq_fops = {
+static const rtw_proc_op_t rtw_adapter_proc_seq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_adapter_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = seq_release,
+   .proc_write = rtw_adapter_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_adapter_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = seq_release,
    .write = rtw_adapter_proc_write,
+#endif
 };

-static const struct file_operations rtw_adapter_proc_sseq_fops = {
+static const rtw_proc_op_t rtw_adapter_proc_sseq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_adapter_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = single_release,
+   .proc_write = rtw_adapter_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_adapter_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = single_release,
    .write = rtw_adapter_proc_write,
+#endif
 };

 int proc_get_odm_adaptivity(struct seq_file *m, void *v)
@@ -4331,22 +4363,38 @@ static ssize_t rtw_odm_proc_write(struct file *file, const char __user *buffer,
    return -EROFS;
 }

-static const struct file_operations rtw_odm_proc_seq_fops = {
+static const rtw_proc_op_t rtw_odm_proc_seq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_odm_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = seq_release,
+   .proc_write = rtw_odm_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_odm_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = seq_release,
    .write = rtw_odm_proc_write,
+#endif
 };

-static const struct file_operations rtw_odm_proc_sseq_fops = {
+static const rtw_proc_op_t rtw_odm_proc_sseq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_odm_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = single_release,
+   .proc_write = rtw_odm_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_odm_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = single_release,
    .write = rtw_odm_proc_write,
+#endif
 };

 struct proc_dir_entry *rtw_odm_proc_init(struct net_device *dev)
@@ -4472,22 +4520,38 @@ static ssize_t rtw_mcc_proc_write(struct file *file, const char __user *buffer,
    return -EROFS;
 }

-static const struct file_operations rtw_mcc_proc_seq_fops = {
+static const rtw_proc_op_t rtw_mcc_proc_seq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_mcc_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = seq_release,
+   .proc_write = rtw_mcc_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_mcc_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = seq_release,
    .write = rtw_mcc_proc_write,
+#endif
 };

-static const struct file_operations rtw_mcc_proc_sseq_fops = {
+static const rtw_proc_op_t rtw_mcc_proc_sseq_fops = {
+#ifdef HAVE_PROC_OPS_STRUCT
+   .proc_open = rtw_mcc_proc_open,
+   .proc_read = seq_read,
+   .proc_lseek = seq_lseek,
+   .proc_release = single_release,
+   .proc_write = rtw_mcc_proc_write,
+#else
    .owner = THIS_MODULE,
    .open = rtw_mcc_proc_open,
    .read = seq_read,
    .llseek = seq_lseek,
    .release = single_release,
    .write = rtw_mcc_proc_write,
+#endif
 };

 struct proc_dir_entry *rtw_mcc_proc_init(struct net_device *dev)
diff --git a/os_dep/linux/rtw_proc.h b/os_dep/linux/rtw_proc.h
index c2c7c8e..839d647 100644
--- a/os_dep/linux/rtw_proc.h
+++ b/os_dep/linux/rtw_proc.h
@@ -21,6 +21,13 @@
 #define RTW_PROC_HDL_TYPE_SEQ  0
 #define RTW_PROC_HDL_TYPE_SSEQ 1

+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+#define HAVE_PROC_OPS_STRUCT
+typedef struct proc_ops rtw_proc_op_t;
+#else
+typedef struct file_operations rtw_proc_op_t;
+#endif
+
 struct rtw_proc_hdl {
    char *name;
    u8 type;
diff --git a/tools/const_structs.checkpatch b/tools/const_structs.checkpatch
index ac5f126..427624c 100644
--- a/tools/const_structs.checkpatch
+++ b/tools/const_structs.checkpatch
@@ -16,6 +16,7 @@ ethtool_ops
 extent_io_ops
 file_lock_operations
 file_operations
+proc_ops
 hv_ops
 ide_dma_ops
 ide_port_ops

---------------- END PATCH --------------

[kimocoder]

Kernel v5.6 patch hasn't been applied yet, it will be added later today.