[FIX] fresh self-hosted instance fails to start up in k8s

Describe the bug

A clear and concise description of what the bug is. Please include what you were expecting to happen vs. what actually happened.

fresh instance of Khoj fails to start up for the first time with the following error

            │ ❱  80 │   │   return executor(sq │
                           │    81 │                          │
                           │    82 │   def _execute(self, sql │
                           │    83 │   │   self.db.validate_n │
                           │                                  │
                           │ /usr/local/lib/python3.10/dist-p │
                           │ ackages/django/db/backends/utils │
                           │ .py:84 in _execute               │
                           │                                  │
                           │    81 │                          │
                           │    82 │   def _execute(self, sql │
                           │    83 │   │   self.db.validate_n │
                           │ ❱  84 │   │   with self.db.wrap_ │
                           │    85 │   │   │   if params is N │
                           │    86 │   │   │   │   # params d │
                           │    87 │   │   │   │   return sel │
                           │                                  │
                           │ /usr/local/lib/python3.10/dist-p │
                           │ ackages/django/db/utils.py:91 in │
                           │ __exit__                         │
                           │                                  │
                           │    88 │   │   │   │   # the conn │
                           │    89 │   │   │   │   if dj_exc_ │
                           │    90 │   │   │   │   │   self.w │
                           │ ❱  91 │   │   │   │   raise dj_e │
                           │    92 │                          │
                           │    93 │   def __call__(self, fun │
                           │    94 │   │   # Note that we are │
                           │                                  │
                           │ /usr/local/lib/python3.10/dist-p │
                           │ ackages/django/db/backends/utils │
                           │ .py:89 in _execute               │
                           │                                  │
                           │    86 │   │   │   │   # params d │
                           │    87 │   │   │   │   return sel │
                           │    88 │   │   │   else:          │
                           │ ❱  89 │   │   │   │   return sel │
                           │    90 │                          │
                           │    91 │   def _executemany(self, │
                           │    92 │   │   self.db.validate_n │
                           ╰──────────────────────────────────╯
                           IntegrityError: null value in column
                           "chat_model_id" of relation
                           "database_agent" violates not-null
                           constraint
                           DETAIL:  Failing row contains (1,
                           2024-03-31 22:27:31.683039+00,
                           2024-03-31 22:27:31.683052+00, Khoj,
                           You are Khoj, a smart, inquisitive
                           and helpful personal assistan...,
                           https://khoj-web-bucket.s3.amazonaws
                           .com/lamp-128.png, ["*"], t, t,
                           khoj, null, null).

To Reproduce

Steps to reproduce the behavior:

apply this k8s yaml

apiVersion: v1
kind: Namespace
metadata:
  name: khoj

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: database
  namespace: khoj
spec:
  selector:
    matchLabels:
      app: database
  serviceName: database
  replicas: 1
  template:
    metadata:
      labels:
        app: database
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
      containers:
        - name: database
          image: pgvector/pgvector:pg16
          env:
            - name: POSTGRES_USER
              value: postgres
            - name: POSTGRES_PASSWORD
              value: postgres
            - name: POSTGRES_DB
              value: postgres
          ports:
            - containerPort: 5432
              name: psql
          volumeMounts:
            - name: data
              mountPath: /var/lib/postgresql/data/
          resources:
            requests:
              memory: "128Mi"
              cpu: "500m"
            limits:
              memory: "2Gi"
              cpu: "1"
          readinessProbe:
            exec:
              command:
                - pg_isready
                - -U
                - postgres
            initialDelaySeconds: 10
            periodSeconds: 30
            timeoutSeconds: 10
            successThreshold: 1
            failureThreshold: 3
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 10Gi
        storageClassName: local-path

# headless svc
---
apiVersion: v1
kind: Service
metadata:
  name: database
  namespace: khoj
spec:
  clusterIP: None
  selector:
    app: database
  ports:
    - port: 5432
      targetPort: 5432

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: khoj
  namespace: khoj
spec:
  selector:
    matchLabels:
      app: khoj
  serviceName: khoj
  replicas: 1
  template:
    metadata:
      labels:
        app: khoj
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
      containers:
        - name: khoj
          image: ghcr.io/khoj-ai/khoj:latest
          ports:
            - containerPort: 42110
              name: web
          volumeMounts:
            - name: config
              mountPath: /root/.khoj
            - name: models
              mountPath: /root/.cache/torch/sentence_transformers
          env:
            - name: POSTGRES_DB
              value: postgres
            - name: POSTGRES_USER
              value: postgres
            - name: POSTGRES_PASSWORD
              value: postgres
            - name: POSTGRES_HOST
              value: database-0.database.khoj.svc.cluster.local
            - name: POSTGRES_PORT
              value: "5432"
            - name: KHOJ_DJANGO_SECRET_KEY
              value: secret
            - name: KHOJ_DEBUG
              value: "False"
            - name: KHOJ_ADMIN_EMAIL
              value: miles.frankel@gmail.com
            - name: KHOJ_ADMIN_PASSWORD
              value: password
          resources:
            requests:
              memory: "1Gi"
              cpu: "100m"
            limits:
              memory: "2Gi"
              cpu: "1"
          args:
            - --host="0.0.0.0"
            - --port=42110
            - -v
            - --anonymous-mode
  volumeClaimTemplates:
    - metadata:
        name: models
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 30Gi
        storageClassName: local-path
    - metadata:
        name: config
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 2Gi
        storageClassName: local-path

Screenshots

If applicable, add screenshots to help explain your problem.

Platform

Server:
- [ ] Cloud-Hosted (https://app.khoj.dev)
- [X] Self-Hosted Docker
- [ ] Self-Hosted Python package
- [ ] Self-Hosted source code
Client:
- [ ] Obsidian
- [ ] Emacs
- [ ] Desktop app
- [ ] Web browser
- [ ] WhatsApp
OS:
- [ ] Windows
- [ ] macOS
- [X] Linux
- [ ] Android
- [ ] iOS

If self-hosted

Server Version [e.g. 1.0.1]:

Additional context

Add any other context about the problem here.

FWIW i get the same error when running the provided docker compose (https://github.com/khoj-ai/khoj/blob/master/docker-compose.yml)

$ docker compose up
...
server-1    | [05:34:06.355199] ERROR    🚨 Failed to configure server on app configure.py:185
server-1    |                            load: null value in column                           
server-1    |                            "chat_model_id" of relation                          
server-1    |                            "database_agent" violates not-null                   
server-1    |                            constraint                                           
server-1    |                            DETAIL:  Failing row contains (1,                    
server-1    |                            2024-04-02 05:34:06.354886+00,                       
server-1    |                            2024-04-02 05:34:06.354896+00, Khoj,                 
server-1    |                            You are Khoj, a smart, inquisitive                   
server-1    |                            and helpful personal assistan...,                    
server-1    |                            https://khoj-web-bucket.s3.amazonaws                 
server-1    |                            .com/lamp-128.png, ["*"], t, t,                      
server-1    |                            khoj, null, null).                                   
server-1    |                                                                                 
server-1    |                            ╭─ Traceback (most recent call las─╮                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/backends/utils │                 
server-1    |                            │ .py:89 in _execute               │                 
server-1    |                            │                                  │                 
server-1    |                            │    86 │   │   │   │   # params d │                 
server-1    |                            │    87 │   │   │   │   return sel │                 
server-1    |                            │    88 │   │   │   else:          │                 
server-1    |                            │ ❱  89 │   │   │   │   return sel │                 
server-1    |                            │    90 │                          │                 
server-1    |                            │    91 │   def _executemany(self, │                 
server-1    |                            │    92 │   │   self.db.validate_n │                 
server-1    |                            ╰──────────────────────────────────╯                 
server-1    |                            NotNullViolation: null value in                      
server-1    |                            column "chat_model_id" of relation                   
server-1    |                            "database_agent" violates not-null                   
server-1    |                            constraint                                           
server-1    |                            DETAIL:  Failing row contains (1,                    
server-1    |                            2024-04-02 05:34:06.354886+00,                       
server-1    |                            2024-04-02 05:34:06.354896+00, Khoj,                 
server-1    |                            You are Khoj, a smart, inquisitive                   
server-1    |                            and helpful personal assistan...,                    
server-1    |                            https://khoj-web-bucket.s3.amazonaws                 
server-1    |                            .com/lamp-128.png, ["*"], t, t,                      
server-1    |                            khoj, null, null).                                   
server-1    |                                                                                 
server-1    |                                                                                 
server-1    |                            The above exception was the direct                   
server-1    |                            cause of the following exception:                    
server-1    |                                                                                 
server-1    |                            ╭─ Traceback (most recent call las─╮                 
server-1    |                            │ /app/src/khoj/configure.py:183   │                 
server-1    |                            │ in initialize_server             │                 
server-1    |                            │                                  │                 
server-1    |                            │   180                            │                 
server-1    |                            │   181 def initialize_server(conf │                 
server-1    |                            │   182 │   try:                   │                 
server-1    |                            │ ❱ 183 │   │   configure_server(c │                 
server-1    |                            │   184 │   except Exception as e: │                 
server-1    |                            │   185 │   │   logger.error(f"🚨  │                 
server-1    |                            │   186                            │                 
server-1    |                            │                                  │                 
server-1    |                            │ /app/src/khoj/configure.py:236   │                 
server-1    |                            │ in configure_server              │                 
server-1    |                            │                                  │                 
server-1    |                            │   233 │   │   setup_default_agen │                 
server-1    |                            │   234 │   │   initialize_content │                 
server-1    |                            │   235 │   except Exception as e: │                 
server-1    |                            │ ❱ 236 │   │   raise e            │                 
server-1    |                            │   237                            │                 
server-1    |                            │   238                            │                 
server-1    |                            │   239 def setup_default_agent(): │                 
server-1    |                            │                                  │                 
server-1    |                            │ /app/src/khoj/configure.py:233   │                 
server-1    |                            │ in configure_server              │                 
server-1    |                            │                                  │                 
server-1    |                            │   230 │   │                      │                 
server-1    |                            │   231 │   │   state.SearchType = │                 
server-1    |                            │   232 │   │   state.search_model │                 
server-1    |                            │       state.config.search_type)  │                 
server-1    |                            │ ❱ 233 │   │   setup_default_agen │                 
server-1    |                            │   234 │   │   initialize_content │                 
server-1    |                            │   235 │   except Exception as e: │                 
server-1    |                            │   236 │   │   raise e            │                 
server-1    |                            │                                  │                 
server-1    |                            │ /app/src/khoj/configure.py:240   │                 
server-1    |                            │ in setup_default_agent           │                 
server-1    |                            │                                  │                 
server-1    |                            │   237                            │                 
server-1    |                            │   238                            │                 
server-1    |                            │   239 def setup_default_agent(): │                 
server-1    |                            │ ❱ 240 │   AgentAdapters.create_d │                 
server-1    |                            │   241                            │                 
server-1    |                            │   242                            │                 
server-1    |                            │   243 def initialize_content(reg │                 
server-1    |                            │       init=False, user: KhojUser │                 
server-1    |                            │                                  │                 
server-1    |                            │ /app/src/khoj/database/adapters/ │                 
server-1    |                            │ __init__.py:451 in               │                 
server-1    |                            │ create_default_agent             │                 
server-1    |                            │                                  │                 
server-1    |                            │   448 │   │   │   agent.save()   │                 
server-1    |                            │   449 │   │   else:              │                 
server-1    |                            │   450 │   │   │   # The default  │                 
server-1    |                            │       little differently than ot │                 
server-1    |                            │ ❱ 451 │   │   │   agent = Agent. │                 
server-1    |                            │   452 │   │   │   │   name=Agent │                 
server-1    |                            │   453 │   │   │   │   public=Tru │                 
server-1    |                            │   454 │   │   │   │   managed_by │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/manager │                 
server-1    |                            │ .py:87 in manager_method         │                 
server-1    |                            │                                  │                 
server-1    |                            │    84 │   │   def create_method( │                 
server-1    |                            │    85 │   │   │   @wraps(method) │                 
server-1    |                            │    86 │   │   │   def manager_me │                 
server-1    |                            │ ❱  87 │   │   │   │   return get │                 
server-1    |                            │    88 │   │   │                  │                 
server-1    |                            │    89 │   │   │   return manager │                 
server-1    |                            │    90                            │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/query.p │                 
server-1    |                            │ y:658 in create                  │                 
server-1    |                            │                                  │                 
server-1    |                            │    655 │   │   """               │                 
server-1    |                            │    656 │   │   obj = self.model( │                 
server-1    |                            │    657 │   │   self._for_write = │                 
server-1    |                            │ ❱  658 │   │   obj.save(force_in │                 
server-1    |                            │    659 │   │   return obj        │                 
server-1    |                            │    660 │                         │                 
server-1    |                            │    661 │   async def acreate(sel │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/base.py │                 
server-1    |                            │ :814 in save                     │                 
server-1    |                            │                                  │                 
server-1    |                            │    811 │   │   │   if loaded_fie │                 
server-1    |                            │    812 │   │   │   │   update_fi │                 
server-1    |                            │    813 │   │                     │                 
server-1    |                            │ ❱  814 │   │   self.save_base(   │                 
server-1    |                            │    815 │   │   │   using=using,  │                 
server-1    |                            │    816 │   │   │   force_insert= │                 
server-1    |                            │    817 │   │   │   force_update= │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/base.py │                 
server-1    |                            │ :877 in save_base                │                 
server-1    |                            │                                  │                 
server-1    |                            │    874 │   │   │   parent_insert │                 
server-1    |                            │    875 │   │   │   if not raw:   │                 
server-1    |                            │    876 │   │   │   │   parent_in │                 
server-1    |                            │ ❱  877 │   │   │   updated = sel │                 
server-1    |                            │    878 │   │   │   │   raw,      │                 
server-1    |                            │    879 │   │   │   │   cls,      │                 
server-1    |                            │    880 │   │   │   │   force_ins │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/base.py │                 
server-1    |                            │ :1020 in _save_table             │                 
server-1    |                            │                                  │                 
server-1    |                            │   1017 │   │   │   │   fields =  │                 
server-1    |                            │   1018 │   │   │                 │                 
server-1    |                            │   1019 │   │   │   returning_fie │                 
server-1    |                            │ ❱ 1020 │   │   │   results = sel │                 
server-1    |                            │   1021 │   │   │   │   cls._base │                 
server-1    |                            │   1022 │   │   │   )             │                 
server-1    |                            │   1023 │   │   │   if results:   │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/base.py │                 
server-1    |                            │ :1061 in _do_insert              │                 
server-1    |                            │                                  │                 
server-1    |                            │   1058 │   │   Do an INSERT. If  │                 
server-1    |                            │   1059 │   │   return the newly  │                 
server-1    |                            │   1060 │   │   """               │                 
server-1    |                            │ ❱ 1061 │   │   return manager._i │                 
server-1    |                            │   1062 │   │   │   [self],       │                 
server-1    |                            │   1063 │   │   │   fields=fields │                 
server-1    |                            │   1064 │   │   │   returning_fie │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/manager │                 
server-1    |                            │ .py:87 in manager_method         │                 
server-1    |                            │                                  │                 
server-1    |                            │    84 │   │   def create_method( │                 
server-1    |                            │    85 │   │   │   @wraps(method) │                 
server-1    |                            │    86 │   │   │   def manager_me │                 
server-1    |                            │ ❱  87 │   │   │   │   return get │                 
server-1    |                            │    88 │   │   │                  │                 
server-1    |                            │    89 │   │   │   return manager │                 
server-1    |                            │    90                            │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/query.p │                 
server-1    |                            │ y:1805 in _insert                │                 
server-1    |                            │                                  │                 
server-1    |                            │   1802 │   │   │   unique_fields │                 
server-1    |                            │   1803 │   │   )                 │                 
server-1    |                            │   1804 │   │   query.insert_valu │                 
server-1    |                            │ ❱ 1805 │   │   return query.get_ │                 
server-1    |                            │   1806 │                         │                 
server-1    |                            │   1807 │   _insert.alters_data = │                 
server-1    |                            │   1808 │   _insert.queryset_only │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/models/sql/com │                 
server-1    |                            │ piler.py:1822 in execute_sql     │                 
server-1    |                            │                                  │                 
server-1    |                            │   1819 │   │   self.returning_fi │                 
server-1    |                            │   1820 │   │   with self.connect │                 
server-1    |                            │   1821 │   │   │   for sql, para │                 
server-1    |                            │ ❱ 1822 │   │   │   │   cursor.ex │                 
server-1    |                            │   1823 │   │   │   if not self.r │                 
server-1    |                            │   1824 │   │   │   │   return [] │                 
server-1    |                            │   1825 │   │   │   if (          │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/backends/utils │                 
server-1    |                            │ .py:67 in execute                │                 
server-1    |                            │                                  │                 
server-1    |                            │    64 │   │   │   │   return sel │                 
server-1    |                            │    65 │                          │                 
server-1    |                            │    66 │   def execute(self, sql, │                 
server-1    |                            │ ❱  67 │   │   return self._execu │                 
server-1    |                            │    68 │   │   │   sql, params, m │                 
server-1    |                            │    69 │   │   )                  │                 
server-1    |                            │    70                            │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │                 
server-1    |                            │ ackages/django/db/backends/utils │                 
server-1    |                            │ .py:80 in _execute_with_wrappers │                 
server-1    |                            │                                  │                 
server-1    |                            │    77 │   │   context = {"connec │                 
server-1    |                            │    78 │   │   for wrapper in rev │                 
server-1    |                            │    79 │   │   │   executor = fun │                 
server-1    |                            │ ❱  80 │   │   return executor(sq │                 
server-1    |                            │    81 │                          │                 
server-1    |                            │    82 │   def _execute(self, sql │                 
server-1    |                            │    83 │   │   self.db.validate_n │                 
server-1    |                            │                                  │                 
server-1    |                            │ /usr/local/lib/python3.10/dist-p │

khoj-ai / khoj