Create gh-proxy

.github/workflows/gh-proxy.yml

@@ -0,0 +1,42 @@
+name: "gh-proxy docker build"
+
+env:
+  PROJECT: gh-proxy
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set tag
+        id: tag
+        run: |
+          TAG=$(cat ${{ env.PROJECT }}/Dockerfile | awk 'NR==4 {print $3}')
+          echo "::set-env name=TAG::$TAG"
+      - name: Docker Hub login
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+      - name: Build Dockerfile
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        run: |
+          docker buildx build \
+          --platform=linux/amd64,linux/arm64 \
+          --output "type=image,push=true" \
+          --file ${{ env.PROJECT }}/Dockerfile ./${{ env.PROJECT }} \
+          --tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:latest \
+          --tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:${TAG}

gh-proxy/Dockerfile

@@ -0,0 +1,28 @@
+FROM stilleshan/uwsgi-nginx:python3.7
+LABEL maintainer="Sebastian Ramirez <[email protected]>"
+
+ENV VERSION 2.1
+
+RUN pip install flask requests
+
+COPY ./app /app
+WORKDIR /app
+
+# Make /app/* available to be imported by Python globally to better support several use cases like Alembic migrations.
+ENV PYTHONPATH=/app
+
+# Move the base entrypoint to reuse it
+RUN mv /entrypoint.sh /uwsgi-nginx-entrypoint.sh
+# Copy the entrypoint that will generate Nginx additional configs
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+# Run the start script provided by the parent image tiangolo/uwsgi-nginx.
+# It will check for an /app/prestart.sh script (e.g. for migrations)
+# And then will start Supervisor, which in turn will start Nginx and uWSGI
+
+EXPOSE 80
+
+CMD ["/start.sh"]

gh-proxy/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 hunshcn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gh-proxy/README.md

@@ -0,0 +1,110 @@
+# gh-proxy
+
+## 简介
+
+github release、archive以及项目文件的加速项目，支持clone，有Cloudflare Workers无服务器版本以及Python版本
+
+## 演示
+
+[https://gh.api.99988866.xyz/](https://gh.api.99988866.xyz/)
+
+演示站为公共服务，如有大规模使用需求请自行部署，演示站有点不堪重负
+
+![imagea272c95887343279.png](https://img.maocdn.cn/img/2021/04/24/imagea272c95887343279.png)
+
+当然也欢迎[捐赠](#捐赠)以支持作者
+
+## python版本和cf worker版本差异
+
+- python版本支持进行文件大小限制，超过设定返回原地址 [issue #8](https://github.com/hunshcn/gh-proxy/issues/8)
+
+## 使用
+
+直接在copy出来的url前加`https://gh.api.99988866.xyz/`即可
+
+也可以直接访问，在input输入
+
+***大量使用请自行部署，以上域名仅为演示使用。***
+
+以下都是合法输入（仅示例，文件不存在）：
+
+- 分支源码：https://github.com/hunshcn/project/archive/master.zip
+
+- release源码：https://github.com/hunshcn/project/archive/v0.1.0.tar.gz
+
+- release文件：https://github.com/hunshcn/project/releases/download/v0.1.0/example.zip
+
+- 分支文件：https://github.com/hunshcn/project/blob/master/filename
+
+- commit文件：https://github.com/hunshcn/project/blob/1111111111111111111111111111/filename
+
+- gist：https://gist.githubusercontent.com/cielpy/351557e6e465c12986419ac5a4dd2568/raw/cmd.py
+
+## cf worker版本部署
+
+首页：https://workers.cloudflare.com
+
+注册，登陆，`Start building`，取一个子域名，`Create a Worker`。
+
+复制 [index.js](https://cdn.jsdelivr.net/hunshcn/gh-proxy@master/index.js)  到左侧代码框，`Save and deploy`。如果正常，右侧应显示首页。
+
+`index.js`默认配置下clone走github.com.cnpmjs.org，项目文件会走jsDeliver，如需走worker，修改Config变量即可
+
+`ASSET_URL`是静态资源的url（实际上就是现在显示出来的那个输入框单页面）
+
+`PREFIX`是前缀，默认（根路径情况为"/"），如果自定义路由为example.com/gh/*，请将PREFIX改为 '/gh/'，注意，少一个杠都会错！
+
+## Python版本部署
+
+### Docker部署
+
+```
+docker run -d --name="gh-proxy-py" \
+  -p 0.0.0.0:80:80 \
+  --restart=always \
+  hunsh/gh-proxy-py:latest
+```
+
+第一个80是你要暴露出去的端口
+
+### 直接部署
+
+安装依赖（请使用python3）
+
+```pip install flask requests```
+
+按需求修改`app/main.py`的前几项配置
+
+### 注意
+
+python版本的机器如果无法正常访问github.io会启动报错，请自行修改静态文件url
+
+workers版本默认配置下clone走github.com.cnpmjs.org，项目文件会走jsDeliver，如需走服务器，修改配置即可
+
+python版本默认走服务器（2021.3.27更新）
+
+## Cloudflare Workers计费
+
+到 `overview` 页面可参看使用情况。免费版每天有 10 万次免费请求，并且有每分钟1000次请求的限制。
+
+如果不够用，可升级到 $5 的高级版本，每月可用 1000 万次请求（超出部分 $0.5/百万次请求）。
+
+## Changelog
+
+* 2020.04.10 增加对`raw.githubusercontent.com`文件的支持
+* 2020.04.09 增加Python版本（使用Flask）
+* 2020.03.23 新增了clone的支持
+* 2020.03.22 初始版本
+
+## 链接
+
+[我的博客](https://hunsh.net)
+
+## 参考
+
+[jsproxy](https://github.com/EtherDream/jsproxy/)
+
+## 捐赠
+
+![wx.png](https://img.maocdn.cn/img/2021/04/24/image.md.png)
+![ali.png](https://www.helloimg.com/images/2021/04/24/BK9vmb.md.png)

gh-proxy/app/main.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+import re
+
+import requests
+from flask import Flask, Response, redirect, request
+from requests.exceptions import (
+    ChunkedEncodingError,
+    ContentDecodingError, ConnectionError, StreamConsumedError)
+from requests.utils import (
+    stream_decode_response_unicode, iter_slices, CaseInsensitiveDict)
+from urllib3.exceptions import (
+    DecodeError, ReadTimeoutError, ProtocolError)
+
+# config
+# git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关，0为关闭，默认关闭
+jsdelivr = 0
+cnpmjs = 0
+size_limit = 1024 * 1024 * 1024 * 999  # 允许的文件大小，默认999GB，相当于无限制了 https://github.com/hunshcn/gh-proxy/issues/8
+HOST = '127.0.0.1'  # 监听地址，建议监听本地然后由web服务器反代
+PORT = 80  # 监听端口
+ASSET_URL = 'https://hunshcn.github.io/gh-proxy'  # 主页
+
+app = Flask(__name__)
+CHUNK_SIZE = 1024 * 10
+index_html = requests.get(ASSET_URL, timeout=10).text
+icon_r = requests.get(ASSET_URL + '/favicon.ico', timeout=10).content
+exp1 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:releases|archive)/.*$')
+exp2 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:blob)/.*$')
+exp3 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:info|git-).*$')
+exp4 = re.compile(r'^(?:https?://)?raw\.githubusercontent\.com/.+?/.+?/.+?/.+$')
+exp5 = re.compile(r'^(?:https?://)?gist\.(?:githubusercontent|github)\.com/.+?/.+?/.+$')
+
+requests.sessions.default_headers = lambda: CaseInsensitiveDict()
+
+
[email protected]('/')
+def index():
+    if 'q' in request.args:
+        return redirect('/' + request.args.get('q'))
+    return index_html
+
+
[email protected]('/favicon.ico')
+def icon():
+    return Response(icon_r, content_type='image/vnd.microsoft.icon')
+
+
+def iter_content(self, chunk_size=1, decode_unicode=False):
+    """rewrite requests function, set decode_content with False"""
+
+    def generate():
+        # Special case for urllib3.
+        if hasattr(self.raw, 'stream'):
+            try:
+                for chunk in self.raw.stream(chunk_size, decode_content=False):
+                    yield chunk
+            except ProtocolError as e:
+                raise ChunkedEncodingError(e)
+            except DecodeError as e:
+                raise ContentDecodingError(e)
+            except ReadTimeoutError as e:
+                raise ConnectionError(e)
+        else:
+            # Standard file-like object.
+            while True:
+                chunk = self.raw.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+
+        self._content_consumed = True
+
+    if self._content_consumed and isinstance(self._content, bool):
+        raise StreamConsumedError()
+    elif chunk_size is not None and not isinstance(chunk_size, int):
+        raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))
+    # simulate reading small chunks of the content
+    reused_chunks = iter_slices(self._content, chunk_size)
+
+    stream_chunks = generate()
+
+    chunks = reused_chunks if self._content_consumed else stream_chunks
+
+    if decode_unicode:
+        chunks = stream_decode_response_unicode(chunks, self)
+
+    return chunks
+
+
[email protected]('/<path:u>', methods=['GET', 'POST'])
+def proxy(u):
+    u = u if u.startswith('http') else 'https://' + u
+    if u.rfind('://', 3, 9) == -1:
+        u = u.replace('s:/', 's://', 1)  # uwsgi会将//传递为/
+    if not any([i.match(u) for i in [exp1, exp2, exp3, exp4, exp5]]):
+        return Response('Invalid input.', status=403)
+    if jsdelivr and exp2.match(u):
+        u = u.replace('/blob/', '@', 1).replace('github.com', 'cdn.jsdelivr.net/gh', 1)
+        return redirect(u)
+    elif cnpmjs and exp3.match(u):
+        u = u.replace('github.com', 'github.com.cnpmjs.org', 1) + request.url.replace(request.base_url, '', 1)
+        return redirect(u)
+    elif jsdelivr and exp4.match(u):
+        u = re.sub(r'(\.com/.*?/.+?)/(.+?/)', r'\1@\2', u, 1)
+        u = u.replace('raw.githubusercontent.com', 'cdn.jsdelivr.net/gh', 1)
+        return redirect(u)
+    else:
+        if exp2.match(u):
+            u = u.replace('/blob/', '/raw/', 1)
+        headers = {}
+        r_headers = dict(request.headers)
+        if 'Host' in r_headers:
+            r_headers.pop('Host')
+        try:
+            url = u + request.url.replace(request.base_url, '', 1)
+            if url.startswith('https:/') and not url.startswith('https://'):
+                url = 'https://' + url[7:]
+            r = requests.request(method=request.method, url=url, data=request.data, headers=r_headers, stream=True)
+            headers = dict(r.headers)
+
+            if 'Content-length' in r.headers and int(r.headers['Content-length']) > size_limit:
+                return redirect(u + request.url.replace(request.base_url, '', 1))
+
+            def generate():
+                for chunk in iter_content(r, chunk_size=CHUNK_SIZE):
+                    yield chunk
+
+            return Response(generate(), headers=headers, status=r.status_code)
+        except Exception as e:
+            headers['content-type'] = 'text/html; charset=UTF-8'
+            return Response('server error ' + str(e), status=500, headers=headers)
+    # else:
+    #     return Response('Illegal input', status=403, mimetype='text/html; charset=UTF-8')
+
+
+if __name__ == '__main__':
+    app.run(host=HOST, port=PORT)

gh-proxy/app/uwsgi.ini

@@ -0,0 +1,3 @@
+[uwsgi]
+module = main
+callable = app

gh-proxy/entrypoint.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+set -e
+
+/uwsgi-nginx-entrypoint.sh
+
+# Get the listen port for Nginx, default to 80
+USE_LISTEN_PORT=${LISTEN_PORT:-80}
+
+if [ -f /app/nginx.conf ]; then
+    cp /app/nginx.conf /etc/nginx/nginx.conf
+else
+    content_server='server {\n'
+    content_server=$content_server"    listen ${USE_LISTEN_PORT};\n"
+    content_server=$content_server'    location / {\n'
+    content_server=$content_server'        try_files $uri @app;\n'
+    content_server=$content_server'    }\n'
+    content_server=$content_server'    location @app {\n'
+    content_server=$content_server'        include uwsgi_params;\n'
+    content_server=$content_server'        uwsgi_pass unix:///tmp/uwsgi.sock;\n'
+    content_server=$content_server'    }\n'
+    content_server=$content_server'}\n'
+    # Save generated server /etc/nginx/conf.d/nginx.conf
+    printf "$content_server" > /etc/nginx/conf.d/nginx.conf
+fi
+
+exec "$@"

gh-proxy/index.js

@@ -0,0 +1,165 @@
+'use strict'
+
+/**
+ * static files (404.html, sw.js, conf.js)
+ */
+const ASSET_URL = 'https://hunshcn.github.io/gh-proxy/'
+// 前缀，如果自定义路由为example.com/gh/*，将PREFIX改为 '/gh/'，注意，少一个杠都会错！
+const PREFIX = '/'
+// git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关，0为关闭，默认开启
+const Config = {
+    jsdelivr: 1,
+    cnpmjs: 1
+}
+
+/** @type {RequestInit} */
+const PREFLIGHT_INIT = {
+    status: 204,
+    headers: new Headers({
+        'access-control-allow-origin': '*',
+        'access-control-allow-methods': 'GET,POST,PUT,PATCH,TRACE,DELETE,HEAD,OPTIONS',
+        'access-control-max-age': '1728000',
+    }),
+}
+
+/**
+ * @param {any} body
+ * @param {number} status
+ * @param {Object<string, string>} headers
+ */
+function makeRes(body, status = 200, headers = {}) {
+    headers['access-control-allow-origin'] = '*'
+    return new Response(body, {status, headers})
+}
+
+
+/**
+ * @param {string} urlStr
+ */
+function newUrl(urlStr) {
+    try {
+        return new URL(urlStr)
+    } catch (err) {
+        return null
+    }
+}
+
+
+addEventListener('fetch', e => {
+    const ret = fetchHandler(e)
+        .catch(err => makeRes('cfworker error:\n' + err.stack, 502))
+    e.respondWith(ret)
+})
+
+
+/**
+ * @param {FetchEvent} e
+ */
+async function fetchHandler(e) {
+    const req = e.request
+    const urlStr = req.url
+    const urlObj = new URL(urlStr)
+    let path = urlObj.searchParams.get('q')
+    if (path) {
+        return Response.redirect('https://' + urlObj.host + PREFIX + path, 301)
+    }
+    // cfworker 会把路径中的 `//` 合并成 `/`
+    path = urlObj.href.substr(urlObj.origin.length + PREFIX.length).replace(/^https?:\/+/, 'https://')
+    const exp1 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:releases|archive)\/.*$/i
+    const exp2 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:blob)\/.*$/i
+    const exp3 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:info|git-).*$/i
+    const exp4 = /^(?:https?:\/\/)?raw\.githubusercontent\.com\/.+?\/.+?\/.+?\/.+$/i
+    const exp5 = /^(?:https?:\/\/)?gist\.(?:githubusercontent|github)\.com\/.+?\/.+?\/.+$/i
+    if (path.search(exp1) === 0 || path.search(exp5) === 0 || !Config.cnpmjs && (path.search(exp3) === 0 || path.search(exp4) === 0)) {
+        return httpHandler(req, path)
+    } else if (path.search(exp2) === 0) {
+        if (Config.jsdelivr){
+            const newUrl = path.replace('/blob/', '@').replace(/^(?:https?:\/\/)?github\.com/, 'https://cdn.jsdelivr.net/gh')
+            return Response.redirect(newUrl, 302)
+        }else{
+            path = path.replace('/blob/', '/raw/')
+            return httpHandler(req, path)
+        }
+    } else if (path.search(exp3) === 0) {
+        const newUrl = path.replace(/^(?:https?:\/\/)?github\.com/, 'https://github.com.cnpmjs.org')
+        return Response.redirect(newUrl, 302)
+    } else if (path.search(exp4) === 0) {
+        const newUrl = path.replace(/(?<=com\/.+?\/.+?)\/(.+?\/)/, '@$1').replace(/^(?:https?:\/\/)?raw\.githubusercontent\.com/, 'https://cdn.jsdelivr.net/gh')
+        return Response.redirect(newUrl, 302)
+    } else {
+        return fetch(ASSET_URL + path)
+    }
+}
+
+
+/**
+ * @param {Request} req
+ * @param {string} pathname
+ */
+function httpHandler(req, pathname) {
+    const reqHdrRaw = req.headers
+
+    // preflight
+    if (req.method === 'OPTIONS' &&
+        reqHdrRaw.has('access-control-request-headers')
+    ) {
+        return new Response(null, PREFLIGHT_INIT)
+    }
+
+    let rawLen = ''
+
+    const reqHdrNew = new Headers(reqHdrRaw)
+
+    let urlStr = pathname
+    if (urlStr.startsWith('github')) {
+        urlStr = 'https://' + urlStr
+    }
+    const urlObj = newUrl(urlStr)
+
+    /** @type {RequestInit} */
+    const reqInit = {
+        method: req.method,
+        headers: reqHdrNew,
+        redirect: 'follow',
+        body: req.body
+    }
+    return proxy(urlObj, reqInit, rawLen, 0)
+}
+
+
+/**
+ *
+ * @param {URL} urlObj
+ * @param {RequestInit} reqInit
+ */
+async function proxy(urlObj, reqInit, rawLen) {
+    const res = await fetch(urlObj.href, reqInit)
+    const resHdrOld = res.headers
+    const resHdrNew = new Headers(resHdrOld)
+
+    // verify
+    if (rawLen) {
+        const newLen = resHdrOld.get('content-length') || ''
+        const badLen = (rawLen !== newLen)
+
+        if (badLen) {
+            return makeRes(res.body, 400, {
+                '--error': `bad len: ${newLen}, except: ${rawLen}`,
+                'access-control-expose-headers': '--error',
+            })
+        }
+    }
+    const status = res.status
+    resHdrNew.set('access-control-expose-headers', '*')
+    resHdrNew.set('access-control-allow-origin', '*')
+
+    resHdrNew.delete('content-security-policy')
+    resHdrNew.delete('content-security-policy-report-only')
+    resHdrNew.delete('clear-site-data')
+
+    return new Response(res.body, {
+        status,
+        headers: resHdrNew,
+    })
+}
+