search

immich-app / immich

High performance self-hosted photo and video management solution.
https://immich.app
GNU Affero General Public License v3.0
33.36k stars 1.6k forks source link

`machine-learning` always download models from huggingface, ignoring `model-cache` #8237

Open shirok1 opened 1 month ago

shirok1 commented 1 month ago

The bug

See the title

The OS that Immich Server is running on

Ubuntu 22.04

Version of Immich Server

v1.99.0

Version of Immich Mobile App

nope

Platform with the issue

Your docker-compose.yml content

version: "3.8"

#
# WARNING: Make sure to use the docker-compose.yml of the current release:
#
# https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml
#
# The compose file on main may not be compatible with the latest release.
#

name: immich

services:
  immich-server:
    container_name: immich_server
    image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
    command: [ "start.sh", "immich" ]
    volumes:
      - ${UPLOAD_LOCATION}:/usr/src/app/upload
      - /drive/archive/family-album:/mnt/media/family-album:ro
      - /etc/localtime:/etc/localtime:ro
    env_file:
      - stack.env
    # ports:
    #   - 2283:3001
    depends_on:
      - redis
      - database
    restart: always
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.immich.rule=Host(`immich.asgzts.shiroki.tech`, `immich.asgz.shiroki.tech`)"
      - "traefik.http.routers.immich.tls=true"
      - "traefik.docker.network=traefik"
    networks:
      - traefik
      - default

  immich-microservices:
    container_name: immich_microservices
    image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
    security_opt: # enables full access to /sys and /proc, still far better than privileged: true
      - systempaths=unconfined
      - apparmor=unconfined
    group_add:
      - video
    devices:
      - /dev/rga:/dev/rga
      - /dev/dri:/dev/dri
      - /dev/dma_heap:/dev/dma_heap
      - /dev/mpp_service:/dev/mpp_service
      - /dev/mali0:/dev/mali0
    command: [ "start.sh", "microservices" ]
    volumes:
      - /etc/OpenCL:/etc/OpenCL:ro
      - /lib/firmware/mali_csffw.bin:/lib/firmware/mali_csffw.bin:ro
      - /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro
      - /usr/lib/aarch64-linux-gnu/mali:/usr/lib/aarch64-linux-gnu/mali:ro
      - ${UPLOAD_LOCATION}:/usr/src/app/upload
      - /drive/archive/family-album:/mnt/media/family-album:ro
      - /etc/localtime:/etc/localtime:ro
    env_file:
      - stack.env
    depends_on:
      - redis
      - database
    restart: always

  immich-machine-learning:
    container_name: immich_machine_learning
    # For hardware acceleration, add one of -[armnn, cuda, openvino] to the image tag.
    # Example tag: ${IMMICH_VERSION:-release}-cuda
    image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}-armnn
    # extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/ml-hardware-acceleration
    #   file: hwaccel.ml.yml
    #   service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference - use the `-wsl` version for WSL2 where applicable
    devices:
      - /dev/rga:/dev/rga
      - /dev/dri:/dev/dri
      - /dev/dma_heap:/dev/dma_heap
      - /dev/mpp_service:/dev/mpp_service
      - /dev/mali0:/dev/mali0
    volumes:
      - model-cache:/cache
      #- /etc/OpenCL:/etc/OpenCL:ro
      - /lib/firmware/mali_csffw.bin:/lib/firmware/mali_csffw.bin:ro
      - /usr/lib/aarch64-linux-gnu/libmali.so:/usr/lib/aarch64-linux-gnu/libmali.so:ro
      - /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro
      - /usr/lib/libmali.so:/usr/lib/libmali.so:ro
      - /usr/lib/aarch64-linux-gnu/mali:/usr/lib/aarch64-linux-gnu/mali:ro
    env_file:
      - stack.env
    environment:
      HTTP_PROXY: "http://192.168.2.1:7890"
      HTTPS_PROXY: "http://192.168.2.1:7890"
      LOG_LEVEL: debug
    restart: always

  redis:
    container_name: immich_redis
    image: redis:6.2-alpine@sha256:51d6c56749a4243096327e3fb964a48ed92254357108449cb6e23999c37773c5
    restart: always

  database:
    container_name: immich_postgres
    image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0
    environment:
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_USER: ${DB_USERNAME}
      POSTGRES_DB: ${DB_DATABASE_NAME}
    volumes:
      - pgdata:/var/lib/postgresql/data
    restart: always

networks:
  traefik:
    external: true

volumes:
  pgdata:
  model-cache:

Your .env content

UPLOAD_LOCATION=/drive/archive/app/immichi
IMMICH_VERSION=release
DB_PASSWORD=CKSVjdMf6AJrifNVDtqx
DB_HOSTNAME=immich_postgres
DB_USERNAME=postgres
DB_DATABASE_NAME=immich
REDIS_HOSTNAME=immich_redis

Reproduction steps

Redeploying the compose with portainer

Additional information

log:

[03/24/24 08:31:08] INFO     Starting gunicorn 21.2.0                           
[03/24/24 08:31:08] INFO     Listening at: http://[::]:3003 (9)                 
[03/24/24 08:31:08] INFO     Using worker: app.config.CustomUvicornWorker       
[03/24/24 08:31:08] INFO     Booting worker with pid: 17                        
[03/24/24 08:31:14] INFO     Started server process [17]                        
[03/24/24 08:31:14] INFO     Waiting for application startup.                   
[03/24/24 08:31:14] INFO     Created in-memory cache with unloading after 300s  
                             of inactivity.                                     
[03/24/24 08:31:14] INFO     Initialized request thread pool with 8 threads.    
[03/24/24 08:31:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:31:14] INFO     Application startup complete.                      
[03/24/24 08:31:14] DEBUG    Available ORT providers: {'CPUExecutionProvider',  
                             'AzureExecutionProvider'}                          
[03/24/24 08:31:14] INFO     Setting 'ViT-B-32__openai' execution providers to  
                             ['CPUExecutionProvider'], in descending order of   
                             preference                                         
[03/24/24 08:31:14] DEBUG    Setting execution provider options to              
                             [{'arena_extend_strategy': 'kSameAsRequested'}]    
[03/24/24 08:31:14] DEBUG    Setting execution_mode to ORT_SEQUENTIAL           
[03/24/24 08:31:14] DEBUG    Setting inter_op_num_threads to 1                  
[03/24/24 08:31:14] DEBUG    Setting intra_op_num_threads to 2                  
[03/24/24 08:31:14] DEBUG    Setting preferred runtime to armnn                 
[03/24/24 08:31:14] DEBUG    Available ORT providers: {'CPUExecutionProvider',  
                             'AzureExecutionProvider'}                          
[03/24/24 08:31:14] INFO     Setting 'buffalo_l' execution providers to         
                             ['CPUExecutionProvider'], in descending order of   
                             preference                                         
[03/24/24 08:31:14] DEBUG    Setting execution provider options to              
                             [{'arena_extend_strategy': 'kSameAsRequested'}]    
[03/24/24 08:31:14] INFO     Downloading clip model 'ViT-B-32__openai'. This may
                             take a while.                                      
[03/24/24 08:31:14] DEBUG    Setting execution_mode to ORT_SEQUENTIAL           
[03/24/24 08:31:14] DEBUG    Setting inter_op_num_threads to 1                  
[03/24/24 08:31:14] DEBUG    Setting intra_op_num_threads to 2                  
[03/24/24 08:31:14] DEBUG    Setting preferred runtime to armnn                 
[03/24/24 08:31:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:31:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:31:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:31:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:32:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:33:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:34:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:35:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:35:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:35:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:35:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:35:44] DEBUG    Checking for inactivity...                         

Fetching 11 files:   0%|          | 0/11 [00:00<?, ?it/s]
Fetching 11 files:   9%|▉         | 1/11 [00:01<00:11,  1.11s/it]
Fetching 11 files:  36%|███▋      | 4/11 [00:01<00:01,  3.56it/s]
Fetching 11 files:  45%|████▌     | 5/11 [03:43<06:12, 62.00s/it]
Fetching 11 files:  91%|█████████ | 10/11 [04:35<00:27, 27.68s/it]
Fetching 11 files: 100%|██████████| 11/11 [04:35<00:00, 25.03s/it]
[03/24/24 08:35:50] INFO     Loading clip model 'ViT-B-32__openai' to memory    
[03/24/24 08:35:50] DEBUG    Loading clip vision model 'ViT-B-32__openai'       
[03/24/24 08:35:50] WARNING  Could not find model path                          
                             '/cache/clip/ViT-B-32__openai/visual/model.armnn'. 
                             Falling back to ONNX model path                    
                             '/cache/clip/ViT-B-32__openai/visual/model.onnx'   
                             instead.                                           
[03/24/24 08:35:51] DEBUG    Loaded clip vision model 'ViT-B-32__openai'        
[03/24/24 08:35:51] DEBUG    Loading tokenizer for CLIP model 'ViT-B-32__openai'
[03/24/24 08:35:51] DEBUG    Loading model config for CLIP model                
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] DEBUG    Loaded model config for CLIP model                 
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] DEBUG    Loading tokenizer config for CLIP model            
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] DEBUG    Loaded tokenizer config for CLIP model             
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] DEBUG    Loaded tokenizer for CLIP model 'ViT-B-32__openai' 
[03/24/24 08:35:51] DEBUG    Loading visual preprocessing config for CLIP model 
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] DEBUG    Loaded visual preprocessing config for CLIP model  
                             'ViT-B-32__openai'                                 
[03/24/24 08:35:51] INFO     Downloading facial recognition model 'buffalo_l'.  
                             This may take a while.                             
[03/24/24 08:35:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:36:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:37:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:37:14] DEBUG    Checking for inactivity...                         

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files:  25%|██▌       | 1/4 [00:01<00:03,  1.04s/it]
Fetching 4 files:  50%|█████     | 2/4 [00:01<00:01,  1.95it/s]
Fetching 4 files:  75%|███████▌  | 3/4 [00:13<00:06,  6.12s/it]
Fetching 4 files: 100%|██████████| 4/4 [01:27<00:00, 32.56s/it]
Fetching 4 files: 100%|██████████| 4/4 [01:27<00:00, 21.77s/it]
[03/24/24 08:37:19] INFO     Loading facial recognition model 'buffalo_l' to    
                             memory                                             
[03/24/24 08:37:19] WARNING  Could not find model path                          
                             '/cache/facial-recognition/buffalo_l/detection/mode
                             l.armnn'. Falling back to ONNX model path          
                             '/cache/facial-recognition/buffalo_l/detection/mode
                             l.onnx' instead.                                   
[03/24/24 08:37:19] WARNING  Could not find model path                          
                             '/cache/facial-recognition/buffalo_l/recognition/mo
                             del.armnn'. Falling back to ONNX model path        
                             '/cache/facial-recognition/buffalo_l/recognition/mo
                             del.onnx' instead.                                 
[03/24/24 08:37:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:37:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:37:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:37:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:44] DEBUG    Checking for inactivity...                         
[03/24/24 08:38:54] DEBUG    Checking for inactivity...                         
[03/24/24 08:39:04] DEBUG    Checking for inactivity...                         
[03/24/24 08:39:14] DEBUG    Checking for inactivity...                         
[03/24/24 08:39:24] DEBUG    Checking for inactivity...                         
[03/24/24 08:39:34] DEBUG    Checking for inactivity...                         
[03/24/24 08:39:45] DEBUG    Checking for inactivity...
mertalev commented 1 month ago

Can you confirm if this happens when you don't try to use ARM NN?

lbatalha commented 3 days ago

I'm also seeing this issue, if I change back to non armnn (default) settings, it does not seem to happen.