`File exists` error

[keilr]

Version

Container image: docker.io/pulp/pulp:3.22 (docker.io/pulp/pulp@sha256:fe90b9131f5d3f4bfe5a2df9cce136b410b5f6d9458c202603470e008675995c)

Python Packages:

$ podman exec pulp pip3 freeze | grep pulp
pulp-python==3.9.0
pulpcore==3.22.19

Describe the bug

We see this error while trying to create a new publication.

pulp [a3b4c85266154415af6e247cc6e792f8]: pulpcore.tasking.pulpcore_worker:INFO: Task 0bf8c1ee-098d-415b-8f4d-728d0349bdab failed ([Errno 17] File exists: 'simple/pillow/')
pulp [a3b4c85266154415af6e247cc6e792f8]: pulpcore.tasking.pulpcore_worker:INFO:   File "/usr/local/lib/python3.8/site-packages/pulpcore/tasking/pulpcore_worker.py", line 460, in execute_task
    result = func(*args, **kwargs)

  File "/usr/local/lib/python3.8/site-packages/pulp_python/app/tasks/publish.py", line 33, in publish
    write_simple_api(pub)

  File "/usr/local/lib/python3.8/site-packages/pulp_python/app/tasks/publish.py", line 88, in write_simple_api
    write_project_page(

  File "/usr/local/lib/python3.8/site-packages/pulp_python/app/tasks/publish.py", line 113, in write_project_page
    os.mkdir(project_dir)

To Reproduce

Repo:

$ pulp python repository list
[
  {
    "pulp_href": "/pulp/api/v3/repositories/python/python/79414063-6324-46e8-8f4a-2eac097a4c83/",
    "pulp_created": "2023-08-15T16:17:20.660892Z",
    "versions_href": "/pulp/api/v3/repositories/python/python/79414063-6324-46e8-8f4a-2eac097a4c83/versions/",
    "pulp_labels": {},
    "latest_version_href": "/pulp/api/v3/repositories/python/python/79414063-6324-46e8-8f4a-2eac097a4c83/versions/118/",
    "name": "pypi",
    "description": null,
    "retain_repo_versions": null,
    "remote": null,
    "autopublish": false
  }
]

Remote:

$ pulp python remote list
[
  {
    "pulp_href": "/pulp/api/v3/remotes/python/python/10d9fa2d-ae94-4565-aee7-5248152526ac/",
    "pulp_created": "2022-08-19T11:13:55.479734Z",
    "name": "pypi",
    "url": https://pypi.org/,
    "ca_cert": null,
    "client_cert": null,
    "tls_validation": true,
    "proxy_url": http://dmz-gate.zrh.local:8080/,
    "pulp_labels": {},
    "pulp_last_updated": "2024-01-30T13:26:39.345748Z",
    "download_concurrency": null,
    "max_retries": null,
    "policy": "on_demand",
    "total_timeout": null,
    "connect_timeout": null,
    "sock_connect_timeout": null,
    "sock_read_timeout": null,
    "headers": null,
    "rate_limit": null,
    "hidden_fields": [
      {
        "name": "client_key",
        "is_set": false
      },
      {
        "name": "proxy_username",
        "is_set": false
      },
      {
        "name": "proxy_password",
        "is_set": false
      },
      {
        "name": "username",
        "is_set": false
      },
      {
        "name": "password",
        "is_set": false
      }
    ],
    "includes": [
      "PySocks",
      "ansible",
      "ansible-base",
      "ansible-builder",
      "ansible-compat",
      "ansible-core",
      "ansible-lint",
      "ansible-navigator",
      "ansible-pylibssh",
      "ansible-runner",
      "anyconfig",
      "appdirs",
      "arrow",
      "asn1crypto",
      "aspy-yaml",
      "atomicwrites",
      "attrs",
      "awxkit",
      "babel",
      "backports-ssl-match-hostname",
      "backports.entry-points-selectable",
      "bcrypt",
      "bindep",
      "binaryornot",
      "black",
      "bracex",
      "build",
      "cerberus",
      "certifi",
      "cffi",
      "cfgv",
      "chardet",
      "charset-normalizer",
      "click",
      "click-completion",
      "click-help-colors",
      "click-shell",
      "colorama",
      "commonmark",
      "configparser",
      "contextlib2",
      "convertdate",
      "cookiecutter",
      "cryptography",
      "dataclasses",
      "debtcollector",
      "defusedxml",
      "distlib",
      "distro",
      "dnspython",
      "docker",
      "docker-py",
      "docker-pycreds",
      "docutils",
      "dparse",
      "enrich",
      "entrypoints",
      "enum34",
      "fasteners",
      "filelock",
      "flake8",
      "funcsigs",
      "functools32",
      "future",
      "git-url-parse",
      "gssapi",
      "hijri-converter",
      "holidays",
      "httpie",
      "identify",
      "idna",
      "importlib-metadata",
      "infoblox-client",
      "iniconfig",
      "ipaddress",
      "iso8601",
      "jinja2",
      "jinja2-time",
      "jmespath",
      "korean-lunar-calendar",
      "krb5",
      "lockfile",
      "lxml",
      "markupsafe",
      "mccabe",
      "molecule",
      "molecule-plugins",
      "monotonic",
      "more-itertools",
      "msgpack",
      "netaddr",
      "netifaces",
      "nodeenv",
      "ntlm-auth",
      "onigurumacffi",
      "oslo-config",
      "oslo-context",
      "oslo-i18n",
      "oslo-log",
      "oslo-serialization",
      "oslo-utils",
      "Parsley",
      "packaging",
      "paramiko",
      "passlib",
      "pathlib2",
      "pathspec",
      "pbr",
      "pep517",
      "pexpect",
      "pip",
      "pip-tools",
      "pipenv",
      "platformdirs",
      "pluggy",
      "poyo",
      "pre-commit",
      "prometheus-client",
      "psutil",
      "ptyprocess",
      "pulp-cli",
      "py",
      "pyasn1",
      "pycodestyle",
      "pycparser",
      "pydf",
      "pyflakes",
      "pygments",
      "pyinotify",
      "pykerberos",
      "pymeeus",
      "pynacl",
      "pyopenssl",
      "pyparsing",
      "pyspnego",
      "pytest",
      "python-daemon",
      "python-dateutil",
      "python-gilt",
      "python-gitlab",
      "python-hpilo",
      "python-memcached",
      "python-slugify",
      "pytz",
      "pyup-tools",
      "pyvmomi",
      "pywinrm",
      "pyyaml",
      "requests",
      "requests-credssp",
      "requests-kerberos",
      "requests-toolbelt",
      "requirements-parser",
      "resolvelib",
      "rfc3986",
      "rich",
      "ruamel-ordereddict",
      "ruamel-yaml",
      "ruamel-yaml-clib",
      "safety",
      "scandir",
      "schema",
      "selinux",
      "setuptools",
      "sh",
      "shellingham",
      "six",
      "stevedore",
      "subprocess-tee",
      "tabulate",
      "tenacity",
      "testinfra",
      "text-unidecode",
      "toml",
      "tomli",
      "tree-format",
      "types-setuptools",
      "typing",
      "typing-extensions",
      "tzdata",
      "urllib3",
      "virtualenv",
      "virtualenv-clone",
      "wcmatch",
      "wcwidth",
      "websocket-client",
      "wheel",
      "whichcraft",
      "wrapt",
      "xmltodict",
      "yamllint",
      "yq",
      "zipp",
      "podman-compose",
      "python-dotenv",
      "absl-py",
      "altair",
      "argon2-cffi",
      "argon2-cffi-bindings",
      "asttokens",
      "astunparse",
      "async-generator",
      "backcall",
      "beautifulsoup4",
      "bleach",
      "bokeh",
      "cachetools",
      "conda",
      "conda-package-handling",
      "cycler",
      "Cython",
      "debugpy",
      "decorator",
      "Delorean",
      "executing",
      "fastjsonschema",
      "fonttools",
      "gast",
      "google-auth",
      "google-auth-oauthlib",
      "google-pasta",
      "grpcio",
      "h5py",
      "humanize",
      "imbalanced-learn",
      "ipykernel",
      "ipynb",
      "ipython",
      "ipython-genutils",
      "ipywidgets",
      "jedi",
      "joblib",
      "json5",
      "jsonschema",
      "jupyter",
      "jupyter-client",
      "jupyter-console",
      "jupyter-core",
      "jupyterlab",
      "jupyterlab-pygments",
      "jupyterlab-server",
      "Keras",
      "Keras-Preprocessing",
      "kiwisolver",
      "Markdown",
      "matplotlib",
      "matplotlib-inline",
      "mistune",
      "nbclient",
      "nbconvert",
      "nbformat",
      "nest-asyncio",
      "notebook",
      "numpy",
      "oauthlib",
      "opt-einsum",
      "pandas",
      "pandocfilters",
      "parso",
      "pickleshare",
      "prompt-toolkit",
      "protobuf",
      "pure-eval",
      "pyasn1-modules",
      "pycosat",
      "pyrsistent",
      "pyzmq",
      "qtconsole",
      "QtPy",
      "requests-oauthlib",
      "rsa",
      "rsconnect-jupyter",
      "rsconnect-python",
      "rsp-jupyter",
      "scikit-learn",
      "scipy",
      "seaborn",
      "semver",
      "Send2Trash",
      "setuptools-scm",
      "sklearn",
      "soupsieve",
      "stack-data",
      "tensorboard",
      "tensorboard-plugin-wit",
      "tensorflow",
      "tensorflow-estimator",
      "termcolor",
      "terminado",
      "testpath",
      "threadpoolctl",
      "tinycss2",
      "toolz",
      "torch",
      "torchvision",
      "tornado",
      "tqdm",
      "traitlets",
      "tzlocal",
      "webencodings",
      "Werkzeug",
      "widgetsnbextension",
      "xgboost"
    ],
    "excludes": [],
    "prereleases": false,
    "package_types": [],
    "keep_latest_packages": 0,
    "exclude_platforms": []
  }
]

Expected behavior

Create new publication

Additional context

This error has been occurring since January 9th. Previously, it worked perfectly, and we didn't make any changes to Pulp.

[keilr]

@gerrod3 FYI

[gerrod3]

@keilr I could not reproduce this with the latest 3.22 image (sha256:b721fe1c300995cdbbda1d5c717d6fdef841dbbb118692b403da8c879159a0af) and the same remote. I think I remember seeing this issue when fixing this: https://github.com/pulp/pulp_python/pull/506. Basically during each task we expect the task to get a new temporary directory to do its work in and that it is cleaned up after the task is finished. The error seems to point to the tmp directory not being cleaned up before the next publication. So I would take a look at your task WORKING_DIRECTORY, defaults to /var/lib/pulp/tmp, and also take a look at the /tmp directory to see if any old files are piling up. If you find any, delete them and then retry the task.

[gerrod3]

@keilr Since there hasn't been any updates in a while I am closing this issue. Feel free to reopen if it has not been resolved.