Closed tawmoto closed 1 year ago
(mind my horrific grammar and spelling, sorry not sorry)
sorry if im late but heres a fun fact dont trust everything you read
i already experienced what you had mate and its not fun
use this Makefile instead: you may inspect it if you want
make sure to set the variables for HOSTNAME and SUBJECT (ive alerady have it in the code below, just replace curl -V -x https://localhost:8899 --proxy-cacert https-cert.pem https://httpbin.org/get
)
SHELL := /bin/bash
NS ?= abhinavsingh
IMAGE_NAME ?= proxy.py
# Override to target specific versions of proxy.py
PROXYPY_CONTAINER_VERSION := latest
# Used by container build and run targets
PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION)
CERT_DIR :=
HTTPS_KEY_FILE_PATH := $(CERT_DIR)https-key.pem
HTTPS_CERT_FILE_PATH := $(CERT_DIR)https-cert.pem
HTTPS_CSR_FILE_PATH := $(CERT_DIR)https-csr.pem
HTTPS_SIGNED_CERT_FILE_PATH := $(CERT_DIR)https-signed-cert.pem
CA_CERT_SUFFIX :=
CA_KEY_FILE_PATH := $(CERT_DIR)ca-key$(CA_CERT_SUFFIX).pem
CA_CERT_FILE_PATH := $(CERT_DIR)ca-cert$(CA_CERT_SUFFIX).pem
CA_SIGNING_KEY_FILE_PATH := $(CERT_DIR)ca-signing-key$(CA_CERT_SUFFIX).pem
HOSTNAME := "<insert hostname>"
SUBJECT := "/CN=<insert hostname>"
# Dummy invalid hardcoded value
PROXYPY_PKG_PATH := dist/proxy.py.whl
BUILDX_TARGET_PLATFORM := linux/amd64
OPEN=$(shell which open)
UNAME := $(shell uname)
ifeq ($(UNAME), Linux)
OPEN=$(shell which xdg-open)
endif
.PHONY: all https-certificates sign-https-certificates ca-certificates
.PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
.PHONY: lib-release-test lib-release lib-profile lib-doc lib-pre-commit
.PHONY: lib-dep lib-flake8 lib-mypy lib-speedscope container-buildx-all-platforms
.PHONY: container container-run container-release container-build container-buildx
.PHONY: devtools dashboard dashboard-clean container-without-openssl
all: lib-test
https-certificates:
# Generate server key
python3 -m proxy.common.pki gen_private_key \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(HTTPS_KEY_FILE_PATH)
python3 -m proxy.common.pki remove_passphrase \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(HTTPS_KEY_FILE_PATH)
# Generate server certificate
python3 -m proxy.common.pki gen_public_key \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(HTTPS_KEY_FILE_PATH) \
--public-key-path $(HTTPS_CERT_FILE_PATH)
sign-https-certificates:
# Generate CSR request
python3 -m proxy.common.pki gen_csr \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--csr-path $(HTTPS_CSR_FILE_PATH) \
--private-key-path $(HTTPS_KEY_FILE_PATH) \
--public-key-path $(HTTPS_CERT_FILE_PATH)
# Sign CSR with CA
python3 -m proxy.common.pki sign_csr \
--subject $(SUBJECT) \
--csr-path $(HTTPS_CSR_FILE_PATH) \
--crt-path $(HTTPS_SIGNED_CERT_FILE_PATH) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_KEY_FILE_PATH) \
--public-key-path $(CA_CERT_FILE_PATH)
ca-certificates:
# Generate CA key
python3 -m proxy.common.pki gen_private_key \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_KEY_FILE_PATH)
python3 -m proxy.common.pki remove_passphrase \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_KEY_FILE_PATH)
# Generate CA certificate
python3 -m proxy.common.pki gen_public_key \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_KEY_FILE_PATH) \
--public-key-path $(CA_CERT_FILE_PATH)
# Generate key that will be used to generate domain certificates on the fly
# Generated certificates are then signed with CA certificate / key generated above
python3 -m proxy.common.pki gen_private_key \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_SIGNING_KEY_FILE_PATH)
python3 -m proxy.common.pki remove_passphrase \
--subject $(SUBJECT) \
--hostname $(HOSTNAME) \
--private-key-path $(CA_SIGNING_KEY_FILE_PATH)
lib-check:
python3 check.py
lib-clean:
find . -name '*.pyc' -exec rm -f {} +
find . -name '*.pyo' -exec rm -f {} +
find . -name '*~' -exec rm -f {} +
rm -f .coverage
rm -rf htmlcov
rm -rf dist
rm -rf build
rm -rf proxy.py.egg-info
rm -rf .pytest_cache
rm -rf .hypothesis
# Doc RST files are cached and can cause issues
# See https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-1003444578
rm -f docs/pkg/*.rst
lib-dep:
pip install --upgrade pip && \
pip install \
-r requirements-testing.txt \
-r requirements-release.txt \
-r requirements-tunnel.txt && \
pip install "setuptools>=42"
lib-pre-commit:
python3 -m pre_commit run --hook-stage manual --all-files -v
lib-lint:
python3 -m tox -e lint
lib-flake8:
tox -e lint -- flake8 --all-files
lib-mypy:
tox -e lint -- mypy --all-files
lib-pytest:
python3 -m tox -e python3 -- -v
lib-test: lib-clean lib-check lib-lint lib-pytest
lib-package: lib-clean lib-check
python3 -m tox -e cleanup-dists,build-dists,metadata-validation
lib-release-test: lib-package
twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*
lib-release: lib-package
twine upload dist/*
lib-doc:
python3 -m tox -e build-docs && \
$(OPEN) .tox/build-docs/docs_out/index.html || true
lib-coverage: lib-clean
pytest --cov=proxy --cov=tests --cov-report=html tests/ && \
$(OPEN) htmlcov/index.html || true
lib-profile:
ulimit -n 65536 && \
sudo py-spy record \
-o profile.svg \
-t -F -s -- \
python3 -m proxy \
--hostname 127.0.0.1 \
--num-acceptors 1 \
--num-workers 1 \
--enable-web-server \
--plugin proxy.plugin.WebServerPlugin \
--backlog 65536 \
--open-file-limit 65536 \
--log-file /dev/null
lib-speedscope:
ulimit -n 65536 && \
sudo py-spy record \
-o profile.speedscope.json \
-f speedscope \
-t -F -s -- \
python3 -m proxy \
--hostname 127.0.0.1 \
--num-acceptors 1 \
--num-workers 1 \
--enable-web-server \
--plugin proxy.plugin.WebServerPlugin \
--backlog 65536 \
--open-file-limit 65536 \
--log-file /dev/null
lib-pyreverse:
rm -f proxy.proxy.Proxy.dot pyreverse.png
pyreverse -ASmy -c proxy.proxy.Proxy proxy
dot -Tpng proxy.proxy.Proxy.dot > pyreverse.png
open pyreverse.png
devtools:
pushd dashboard && npm install && npm run devtools && popd
dashboard:
pushd dashboard && npm install && npm run build && popd
dashboard-clean:
if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi
container: lib-package
docker build \
-t $(PROXYPY_CONTAINER_TAG) \
--build-arg PROXYPY_PKG_PATH=$$(ls dist/*.whl) .
container-without-openssl: lib-package
docker build \
-t $(PROXYPY_CONTAINER_TAG) \
--build-arg SKIP_OPENSSL=1 \
--build-arg PROXYPY_PKG_PATH=$$(ls dist/*.whl) .
# Usage:
#
# make container-buildx \
# -e PROXYPY_PKG_PATH=$(ls dist/*.whl) \
# -e BUILDX_TARGET_PLATFORM=linux/arm64 \
# -e PROXYPY_CONTAINER_VERSION=latest
container-buildx:
docker buildx build \
--load \
--platform $(BUILDX_TARGET_PLATFORM) \
-t $(PROXYPY_CONTAINER_TAG) \
--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
container-buildx-all-platforms:
docker buildx build \
--platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x \
-t $(PROXYPY_CONTAINER_TAG) \
--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
container-run:
docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG)
@EpicBirb @tawmoto I guess TLSV1_ALERT_UNKNOWN_CA
is expected for self signed certificates? proxy
uses a CA file for verification. It defaults to certifi/cacert.pem
but you can also override it by using --ca-file
flag.
@EpicBirb @tawmoto I guess
TLSV1_ALERT_UNKNOWN_CA
is expected for self signed certificates?proxy
uses a CA file for verification. It defaults tocertifi/cacert.pem
but you can also override it by using--ca-file
flag.
maybe i will try that because i have been fiddling around by setting up end to end encryption without passing the --proxy-cacert flag to curl
@EpicBirb @tawmoto I guess
TLSV1_ALERT_UNKNOWN_CA
is expected for self signed certificates?proxy
uses a CA file for verification. It defaults tocertifi/cacert.pem
but you can also override it by using--ca-file
flag.
looks like overriding it didn't work
@abhinavsingh thank you for the reply. I just want to connect to a server, any server, via proxy.py https.
So smth like this
curl -V -x <proxy.py https server> --proxy-cacert https-cert.pem https://httpbin.org/get
Can you tell me the exact steps i can achieve this in the simplest possible way?
Thank you
@EpicBirb @tawmoto I guess
TLSV1_ALERT_UNKNOWN_CA
is expected for self signed certificates?proxy
uses a CA file for verification. It defaults tocertifi/cacert.pem
but you can also override it by using--ca-file
flag.
![]()
looks like overriding it didn't work
scrap that i had the wrong hostname and subject
the more i look the more i regret
@abhinavsingh thank you for the reply. I just want to connect to a server, any server, via proxy.py https.
So smth like this
curl -V -x <proxy.py https server> --proxy-cacert https-cert.pem https://httpbin.org/get
Can you tell me the exact steps i can achieve this in the simplest possible way?
Thank you
curl
and --proxy-cacert https-cert.pem
flag?Let me know
@abhinavsingh I am trying to follow the instructions on your site, exactly
so on the server side I get
$ ~/Desktop/proxy.py/proxy.py$ proxy \
> --cert-file https-cert.pem \
> --key-file https-key.pem
2022-11-01 13:31:11,609 - pid:1927364 [I] plugins.load:85 - Loaded plugin proxy.http.proxy.HttpProxyPlugin
2022-11-01 13:31:11,609 - pid:1927364 [I] tcp.listen:80 - Listening on 127.0.0.1:8899
2022-11-01 13:31:11,622 - pid:1927364 [I] pool.setup:105 - Started 8 acceptors in threadless (local) mode
2022-11-01 13:31:19,749 - pid:1927373 [E] fd.work:48 - Exception occurred during initialization
Traceback (most recent call last):
File "/home/tawfic/.local/lib/python3.8/site-packages/proxy/core/work/fd/fd.py", line 45, in work
self.works[fileno].initialize()
File "/home/tawfic/.local/lib/python3.8/site-packages/proxy/http/handler.py", line 63, in initialize
super().initialize()
File "/home/tawfic/.local/lib/python3.8/site-packages/proxy/core/base/tcp_server.py", line 129, in initialize
conn = self._optionally_wrap_socket(self.work.connection)
File "/home/tawfic/.local/lib/python3.8/site-packages/proxy/core/base/tcp_server.py", line 240, in _optionally_wrap_socket
conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile)
File "/home/tawfic/.local/lib/python3.8/site-packages/proxy/common/utils.py", line 232, in wrap_socket
return ctx.wrap_socket(
File "/usr/lib/python3.8/ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "/usr/lib/python3.8/ssl.py", line 1040, in _create
self.do_handshake()
File "/usr/lib/python3.8/ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:1131)
client side
curl -x https://localhost:8899 --proxy-cacert https-cert.pem https://httpbin.org/get
curl: (77) error setting certificate verify locations:
CAfile: https-cert.pem
CApath: /etc/ssl/certs
btw I am using the master
branch
Simplest is to trust your certificate in your system keychain, which will then make this work without a flag.
Unfortunately, his is not an option for me, I have to test that my product works with a custom certificate set by my customers
@tawmoto Can you please also try with the develop
branch once.
PS: TLS decryption for httpbin.org
is part of our integration test suite. Every code merged in develop
passes these tests that you are trying. So, I believe its mostly about setup issues.
Meanwhile, I'll also try to double check on it locally once.
@abhinavsingh yep, works perfectly now, after using develop
, thanks
curl -x https://localhost:8899 --proxy-cacert https-cert.pem https://httpbin.org/get
{
"args": {},
"headers": {
"Accept": "*/*",
"Host": "httpbin.org",
"User-Agent": "curl/7.68.0",
"X-Amzn-Trace-Id": "Root=1-63612b95-030da4ac71ef26447682ead0"
},
"origin": "86.126.30.14",
"url": "https://httpbin.org/get"
}
thanks a lot!
@tawmoto I'll archive/delete the master
branch as I no longer try to keep it upto-date. Sorry for the inconvenience. Happy it worked out for you :)
@EpicBirb could you also try with develop
and confirm. Please re-open if issue still persists.
@abhinavsingh thanks Maybe work on master, usually master is the main branch, but it's your call, ofc :)
Hi, I want to set proxy.py with end-to-end-encryption.
I cloned the repo and I did this:
then this
and when I try to connect to it via
curl
nothing happensand when i try with another software, i get this
Version information
thank you