search

caddyserver / replace-response

Caddy module that performs replacements in response bodies
Apache License 2.0
103 stars 27 forks source link

Questions about configuration #32

Closed nitezs closed 1 month ago

nitezs commented 1 month ago

I'm trying to use the configuration below mirror github, but the links in the page are not replaced, and I can't find the reason at all.

{
    order replace after encode
}

(communal) {
    header_down -Strict-Transport-Security
    header_down -Content-Security-Policy
    header_down -Set-Cookie
    header_down -x-pjax-url
    header_down -Referrer-Policy
    header_up Accept-Encoding identity
}

g.example.com {
    import log g.example.com
    route {
        reverse_proxy https://github.com {
            import communal
            header_up Host github.com
            header_up Referer https://github.com/
            header_up Origin https://github.com
        }
        replace {
            raw.githubusercontent.com g-raw.example.com
            github.com g.example.com
            github.githubassets.com g-assets.example.com
            api.github.com g-api.example.com
        }
        @downloads path_regexp downloads ^/[^/]+/[^/]+/releases/download/
        redir @downloads https://g-download.example.com{re.download.0}

        @archives path_regexp archives ^/[^/]+/[^/]+/archive/
        redir @archives https://g-archive.example.com{re.archives.0}

        @artifacts path_regexp artifacts ^/[^/]+/[^/]+/suites/[^/]+/artifacts/
        redir @artifacts https://g-download.example.com{re.artifacts.0}
    }
}

g-assets.example.com {
    reverse_proxy https://github.githubassets.com {
        import communal

        header_up Host github.githubassets.com
    }
}

g-codeload.example.com {
    reverse_proxy https://codeload.github.com {
        import communal

        header_up Host github.com
    }
}

g-download.example.com {
    @downloads path_regexp downloads ^/[^/]+/[^/]+/releases(/latest)?/download/
    reverse_proxy @downloads https://github.com {
        import communal

        header_up Host github.com
    }

    @artifacts path_regexp artifacts ^/[^/]+/[^/]+/suites/[^/]+/artifacts/
    reverse_proxy @artifacts https://github.com {
        import communal

        header_up Host github.com
    }

    @archives path_regexp archives ^/[^/]+/[^/]+/archive/
    redir @archives https://g-archive.example.com{re.archives.0}
}

g-raw.example.com {
    route {
        reverse_proxy https://raw.githubusercontent.com {
            import communal

            header_up Host raw.githubusercontent.com
        }
        replace raw.githubusercontent.com g-raw.example.com
    }
}

g-api.example.com {
    import log g-api.example.com
    route {
        reverse_proxy https://api.github.com {
            import communal

            header_up Host api.github.com
        }
        replace {
            github.com g.example.com
            raw.githubusercontent.com g-raw.example.com
            api.github.com g-api.example.com
        }
    }
}

g-archive.example.com {
    @downloads path_regexp downloads ^/[^/]+/[^/]+/releases(/latest)?/download/
    redir @downloads https://g-download.example.com{re.downloads.0}

    @artifacts path_regexp artifacts ^/[^/]+/[^/]+/suites/[^/]+/artifacts/
    redir @artifacts https://g-download.example.com{re.artifacts.0}

    @archives path_regexp archives ^/[^/]+/[^/]+/archive/
    reverse_proxy @archives https://github.com {
        import communal
        header_up Host github.com
    }
}
nitezs commented 1 month ago

Debug logs

{
  "level": "debug",
  "ts": 1728500030.6643522,
  "logger": "http.handlers.reverse_proxy",
  "msg": "upstream roundtrip",
  "upstream": "github.com:443",
  "duration": 0.144535144,
  "request": {
    "remote_ip": "",
    "remote_port": "16764",
    "client_ip": "",
    "proto": "HTTP/3.0",
    "method": "GET",
    "host": "github.com",
    "uri": "/nitezs/sub2sing-box/security/overall-count",
    "headers": {
      "Accept-Encoding": [""],
      "X-Requested-With": ["XMLHttpRequest"],
      "Sec-Fetch-Site": ["same-origin"],
      "Sec-Fetch-Dest": ["empty"],
      "User-Agent": [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
      ],
      "Cookie": ["REDACTED"],
      "Origin": ["https://github.com"],
      "Pragma": ["no-cache"],
      "Accept-Language": ["zh-CN,zh;q=0.9,en-GB;q=0.8,en;q=0.7,zh-TW;q=0.6"],
      "X-Forwarded-Proto": ["https"],
      "Sec-Fetch-Mode": ["cors"],
      "Priority": ["u=1, i"],
      "Sec-Ch-Ua-Mobile": ["?0"],
      "X-Forwarded-Host": ["g.example.com"],
      "Sec-Ch-Ua-Platform": ["\"Windows\""],
      "Cache-Control": ["no-cache"],
      "Sec-Ch-Ua": [
        "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\""
      ],
      "X-Forwarded-For": [""],
      "Referer": ["https://github.com/"],
      "Accept": ["text/fragment+html"]
    },
    "tls": {
      "resumed": true,
      "version": 772,
      "cipher_suite": 4865,
      "proto": "h3",
      "server_name": "g.example.com"
    }
  },
  "headers": {
    "Content-Security-Policy": [
      "default-src 'none'; base-uri 'self'; child-src github.com/assets-cdn/worker/ github.com/webpack/ github.com/assets/ gist.github.com/assets-cdn/worker/; connect-src 'self' uploads.github.com www.githubstatus.com collector.github.com raw.githubusercontent.com api.github.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com *.rel.tunnels.api.visualstudio.com wss://*.rel.tunnels.api.visualstudio.com objects-origin.githubusercontent.com copilot-proxy.githubusercontent.com proxy.individual.githubcopilot.com proxy.business.githubcopilot.com proxy.enterprise.githubcopilot.com *.actions.githubusercontent.com wss://*.actions.githubusercontent.com productionresultssa0.blob.core.windows.net/ productionresultssa1.blob.core.windows.net/ productionresultssa2.blob.core.windows.net/ productionresultssa3.blob.core.windows.net/ productionresultssa4.blob.core.windows.net/ productionresultssa5.blob.core.windows.net/ productionresultssa6.blob.core.windows.net/ productionresultssa7.blob.core.windows.net/ productionresultssa8.blob.core.windows.net/ productionresultssa9.blob.core.windows.net/ productionresultssa10.blob.core.windows.net/ productionresultssa11.blob.core.windows.net/ productionresultssa12.blob.core.windows.net/ productionresultssa13.blob.core.windows.net/ productionresultssa14.blob.core.windows.net/ productionresultssa15.blob.core.windows.net/ productionresultssa16.blob.core.windows.net/ productionresultssa17.blob.core.windows.net/ productionresultssa18.blob.core.windows.net/ productionresultssa19.blob.core.windows.net/ github-production-repository-image-32fea6.s3.amazonaws.com github-production-release-asset-2e65be.s3.amazonaws.com insights.github.com wss://alive.github.com api.githubcopilot.com api.individual.githubcopilot.com api.business.githubcopilot.com api.enterprise.githubcopilot.com; font-src github.githubassets.com; form-action 'self' github.com gist.github.com copilot-workspace.githubnext.com objects-origin.githubusercontent.com; frame-ancestors 'none'; frame-src viewscreen.githubusercontent.com notebooks.githubusercontent.com; img-src 'self' data: blob: github.githubassets.com media.githubusercontent.com camo.githubusercontent.com identicons.github.com avatars.githubusercontent.com private-avatars.githubusercontent.com github-cloud.s3.amazonaws.com objects.githubusercontent.com secured-user-images.githubusercontent.com/ user-images.githubusercontent.com/ private-user-images.githubusercontent.com opengraph.githubassets.com github-production-user-asset-6210df.s3.amazonaws.com customer-stories-feed.github.com spotlights-feed.github.com objects-origin.githubusercontent.com *.githubusercontent.com; manifest-src 'self'; media-src github.com user-images.githubusercontent.com/ secured-user-images.githubusercontent.com/ private-user-images.githubusercontent.com github-production-user-asset-6210df.s3.amazonaws.com gist.github.com; script-src github.githubassets.com; style-src 'unsafe-inline' github.githubassets.com; upgrade-insecure-requests; worker-src github.com/assets-cdn/worker/ github.com/webpack/ github.com/assets/ gist.github.com/assets-cdn/worker/"
    ],
    "Accept-Ranges": ["bytes"],
    "X-Github-Request-Id": ["B1AA:14C27F:396E177:3A64919:6706D13E"],
    "Content-Type": ["text/fragment+html; charset=utf-8"],
    "Vary": [
      "X-PJAX, X-PJAX-Container, Turbo-Visit, Turbo-Frame, Accept-Encoding, Accept, X-Requested-With"
    ],
    "X-Frame-Options": ["deny"],
    "Referrer-Policy": ["no-referrer-when-downgrade"],
    "X-Content-Type-Options": ["nosniff"],
    "X-Xss-Protection": ["0"],
    "Server": ["GitHub.com"],
    "Date": ["Wed, 09 Oct 2024 18:53:50 GMT"],
    "Cache-Control": ["max-age=14400, private"],
    "Strict-Transport-Security": [
      "max-age=31536000; includeSubdomains; preload"
    ]
  },
  "status": 200
}
francislavoie commented 1 month ago

You have your directives in a route, which overrides directive order.

replace needs to be before reverse_proxy. It's a streaming filter which needs to be set up before the proxy runs.

reverse_proxy is a terminal handler, which means nothing after reverse_proxy is run (unless you use handle_response inside it, but that's besides the point).

You already use the order global option, so you can simply get rid of the route wrapper, which would let the Caddyfile adapter sort your directives correctly.

@downloads path_regexp downloads ^/[^/]+/[^/]+/releases/download/
redir @downloads https://g-download.example.com/{re.download.0}

@archives path_regexp archives ^/[^/]+/[^/]+/archive/
redir @archives https://g-archive.example.com/{re.archives.0}

@artifacts path_regexp artifacts ^/[^/]+/[^/]+/suites/[^/]+/artifacts/
redir @artifacts https://g-download.example.com/{re.artifacts.0}

As of v2.8.0, you can simplify this (drop the regexp name):

@downloads path_regexp ^/[^/]+/[^/]+/releases/download/
redir @downloads https://g-download.example.com/{re.0}

@archives path_regexp ^/[^/]+/[^/]+/archive/
redir @archives https://g-archive.example.com/{re.0}

@artifacts path_regexp ^/[^/]+/[^/]+/suites/[^/]+/artifacts/
redir @artifacts https://g-download.example.com/{re.0}
nitezs commented 1 month ago

It worked, thank you very much for your help! 😄