proposal: open github discussion for expert user / anyone

[kolinfluence]

Proposal Details

i understand there's stackoverflow but it's limited etc but if your question is an extreme case / expert level and no one can answer the question, you start to get a lot of very simplistic answer.

Possible to open up the discussion with a section for extreme golang / expert golang dev only? I think with 5k+ issues, if you open up the discussion, some simple questions can be asked there instead of having so many issues created.

i asked for multi process inter process global mutex lock in telegram etc.

my question was, how to get the following to run or is there a way which is supported by golang? i realised i cant run > 80k go routines over cgo without crashing even if i set ulimit -n 1024000 or setmaxthreads 1024000 too. so not sure how to push to make it work. need inter process global mutex lock between golang processes for accessing mmap file contents.

the below doesnt work if the goroutine is set higher to maybe 10000 or if the program is ran multiple times on the same hosts.

i've tried everything including golang futex etc and using other's packages and none works without crashing.

package main

/*
#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

// Helper function to get the current errno value
int get_errno() {
    return errno;
}

// Function to initialize a mutex as process-shared
int setupMutex(void* addr) {
    pthread_mutexattr_t attr;
    pthread_mutex_t* mutex = (pthread_mutex_t*) addr;

    if (pthread_mutexattr_init(&attr)) {
        perror("Mutex attribute init failed");
        return -1;
    }

    if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) {
        perror("Setting pshared failed");
        pthread_mutexattr_destroy(&attr);
        return -1;
    }

    if (pthread_mutex_init(mutex, &attr)) {
        perror("Mutex init failed");
        pthread_mutexattr_destroy(&attr);
        return -1;
    }

    pthread_mutexattr_destroy(&attr);
    return 0;
}

// Function to create shared memory with error handling
int createSharedMemory(key_t key, size_t size) {
    int shmid = shmget(key, size, IPC_CREAT | 0666);
    if (shmid == -1) {
        fprintf(stderr, "Failed to create/find shared memory: %s\n", strerror(errno));
    }
    return shmid;
}
*/
import "C"

import (
    "log"
    "sync"
    "sync/atomic"
    "time"
    "unsafe"
)

var (
    lockUnlockCount int64
    reportFrequency = time.Second
)

func main() {
    path := C.CString("/dev/null") // Use an accessible path on your system
    defer C.free(unsafe.Pointer(path))
    id := C.int(123) // Arbitrary project identifier

    key := C.ftok(path, id)
    if key == -1 {
        log.Fatalf("ftok failed: %s", C.GoString(C.strerror(C.get_errno())))
    }

    size := C.size_t(unsafe.Sizeof(C.pthread_mutex_t{}))
    shmid := C.createSharedMemory(key, size)
    if shmid < 0 {
        log.Fatal("Exiting due to previous error.")
    }

    shmaddr := C.shmat(shmid, nil, 0)
    if shmaddr == nil {
        log.Fatalf("Failed to attach shared memory: %s", C.GoString(C.strerror(C.get_errno())))
    }
    defer C.shmdt(shmaddr)

    if C.setupMutex(shmaddr) != 0 {
        log.Fatalf("Failed to setup mutex")
    }

    var wg sync.WaitGroup
    go reportLockUnlockOperations()

    const numGoroutines = 100
    wg.Add(numGoroutines)

    for i := 0; i < numGoroutines; i++ {
        go func() {
            defer wg.Done()
            for {
                if C.pthread_mutex_lock((*C.pthread_mutex_t)(shmaddr)) != 0 {
                    log.Println("Failed to lock mutex")
                    continue
                }
                atomic.AddInt64(&lockUnlockCount, 1)
                if C.pthread_mutex_unlock((*C.pthread_mutex_t)(shmaddr)) != 0 {
                    log.Println("Failed to unlock mutex")
                    continue
                }
            }
        }()
    }

    wg.Wait()
}

func reportLockUnlockOperations() {
    for range time.Tick(reportFrequency) {
        count := atomic.SwapInt64(&lockUnlockCount, 0)
        log.Printf("Lock-unlock operations per second: %d", count)
    }
}

[kolinfluence]

i've tried even this, which doesnt run on 2nd instance on same host:

package main

import (
    "log"
    "sync"
    "sync/atomic"
    "time"

    "modernc.org/libc"
)

var (
    tls             = libc.NewTLS()
    lockUnlockCount int64
    reportFrequency = time.Second
)

func main() {
    const key, size, mode = 2, 4, libc.IPC_CREAT | 0666
    shmid := libc.Xshmget(tls, key, size, mode)
    if shmid < 0 {
        log.Fatalf("Failed to create/find shared memory: %d", shmid)
    }

    shmaddr := libc.Xshmat(tls, shmid, 0, 0)
    if int64(shmaddr) < 0 {
        log.Fatalf("Failed to attach shared memory: %d", int64(shmaddr))
    }
    defer libc.Xshmdt(tls, shmaddr)

    setupMutex(tls, shmaddr)  // Set up the mutex properly

    var wg sync.WaitGroup
    go reportLockUnlockOperations()

    const numGoroutines = 100
    wg.Add(numGoroutines)

    for i := 0; i < numGoroutines; i++ {
        go func() {
            defer wg.Done()
            for {
                if err := libc.Xpthread_mutex_lock(tls, shmaddr); err != 0 {
                    log.Printf("Failed to lock mutex: %d", err)
                    continue
                }
                atomic.AddInt64(&lockUnlockCount, 1)
                if err := libc.Xpthread_mutex_unlock(tls, shmaddr); err != 0 {
                    log.Printf("Failed to unlock mutex: %d", err)
                    continue
                }
            }
        }()
    }

    wg.Wait()
}

func setupMutex(tls *libc.TLS, addr uintptr) {
    // Assume mutexattr is set up elsewhere to configure the mutex as robust and process-shared
    if err := libc.Xpthread_mutex_init(tls, addr, 0); err != 0 {
        log.Fatalf("Failed to initialize mutex: %d", err)
    }
}

func reportLockUnlockOperations() {
    for range time.Tick(reportFrequency) {
        count := atomic.SwapInt64(&lockUnlockCount, 0)
        log.Printf("Lock-unlock operations per second: %d", count)
    }
}

[ALTree]

We experimented with allowing questions on the issue tracker but it didn't really work and we decided to keep the github issue tracker strictly for bug reports and change proposals; it seems unlikely that this decision will be reverted.

I understand asking a question and receiving no answer can be frustrating, but the github issue tracker is for bugs only, so I'm going to close this.

[ianlancetaylor]

Yeah, it's not that we don't want to help, it's that in practice questions went unanswered which was a poor user experience. The forums (https://go.dev/wiki/Questions) really are better places to ask complex questions.