add kubernetes

Dockerfile

@@ -1,9 +1,13 @@
-FROM python:3.6-alpine
+FROM nginx:alpine
 WORKDIR /app
+RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev \
+    openssl-dev libxml2-dev libxslt-dev gcc musl-dev py3-pip && \
+    apk add --no-cache --update python3 && \
+    pip3 install --upgrade pip setuptools
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt && \
+apk del g++ gcc musl-dev libxml2-dev
 COPY . .
 # RUN pip install -r requirements.txt  -i https://pypi.douban.com/simple
-RUN apk add --no-cache libxml2-dev libxslt-dev gcc musl-dev && \
-pip install -r requirements.txt && \
-apk del gcc musl-dev libxml2-dev
 VOLUME ["/app/proxypool/crawlers/private"]
 CMD ["supervisord", "-c", "supervisord.conf"]

README.md

@@ -341,11 +341,7 @@ class Daili66Crawler(BaseCrawler):
 
 ## 部署
 
-本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，执行如下命令即可：
-
-```shell script
-cat deployment.yml | sed 's/\${TAG}/latest/g' | kubectl apply -f -
-```
+本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，请参考 [kubernetes](./kubernetes)。
 
 ## 待开发
 

build.yml → build.yaml

@@ -8,7 +8,7 @@ services:
     # restart: always
   proxypool:
     build: .
-    image: "germey/proxypool"
+    image: "germey/proxypool:master"
     container_name: proxypool
     ports:
       - "5555:5555"

deployment.yml

@@ -1,99 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  creationTimestamp: null
-  name: proxypool
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: proxypool
-  namespace: proxypool
-spec:
-  storageClassName: azure-file
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 2Gi
----
-apiVersion: v1
-items:
-  - apiVersion: v1
-    kind: Service
-    metadata:
-      annotations:
-        kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-        kompose.version: 1.20.0 ()
-      creationTimestamp: null
-      labels:
-        io.kompose.service: proxypool
-      name: proxypool
-      namespace: proxypool
-    spec:
-      ports:
-        - name: "5555"
-          port: 5555
-          targetPort: 5555
-      selector:
-        io.kompose.service: proxypool
-    status:
-      loadBalancer: {}
-  - apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      annotations:
-        kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-        kompose.version: 1.20.0 ()
-      creationTimestamp: null
-      labels:
-        io.kompose.service: proxypool
-      name: proxypool
-      namespace: proxypool
-    spec:
-      replicas: 2
-      revisionHistoryLimit: 1
-      strategy: {}
-      selector:
-        matchLabels:
-          io.kompose.service: proxypool
-      template:
-        metadata:
-          annotations:
-            kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-            kompose.version: 1.20.0 ()
-          creationTimestamp: null
-          labels:
-            io.kompose.service: proxypool
-        spec:
-          containers:
-            - env:
-                - name: REDIS_CONNECTION_STRING
-                  valueFrom:
-                    secretKeyRef:
-                      name: redis
-                      key: connection_string
-                - name: REDIS_PORT
-                  value: '6379'
-              image: germey/proxypool:${TAG}
-              name: proxypool
-              resources:
-                limits:
-                  memory: "500Mi"
-                  cpu: "300m"
-                requests:
-                  memory: "500Mi"
-                  cpu: "300m"
-              ports:
-                - containerPort: 5555
-              volumeMounts:
-                - mountPath: "/app/proxypool/crawlers/private"
-                  name: proxypool
-          restartPolicy: Always
-          volumes:
-            - name: proxypool
-              persistentVolumeClaim:
-                claimName: pvc-proxypool
-    status: {}
-kind: List
-metadata: {}

ingress.yml

@@ -1,32 +0,0 @@
----
-apiVersion: networking.k8s.io/v1beta1
-kind: Ingress
-metadata:
-  name: ingress-universal-proxypool
-  namespace: proxypool
-  annotations:
-    nginx.ingress.kubernetes.io/ssl-redirect: "true"
-    nginx.ingress.kubernetes.io/rewrite-target: /
-spec:
-  tls:
-    - hosts:
-        - universal.proxypool.cuiqingcai.com
-      secretName: tls-wildcard-proxypool-cuiqingcai-com
-    - hosts:
-        - proxypool.scrape.center
-      secretName: tls-wildcard-scrape-center
-  rules:
-    - host: universal.proxypool.cuiqingcai.com
-      http:
-        paths:
-          - backend:
-              serviceName: proxypool
-              servicePort: 5555
-            path: /
-    - host: proxypool.scrape.center
-      http:
-        paths:
-          - backend:
-              serviceName: proxypool
-              servicePort: 5555
-            path: /

kubernetes/.helmignore

@@ -0,0 +1,24 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+image/

kubernetes/Chart.yaml

@@ -0,0 +1,27 @@
+apiVersion: v2
+name: proxypool
+description: A Efficient Proxy Pool
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# Keywords about this application.
+keywords:
+  - proxypool
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 1.16.0

kubernetes/README.md

@@ -0,0 +1,37 @@
+# Kubernetes 部署
+
+这是用来快速部署本代理池的 Helm Charts。
+
+首先需要有一个 Kubernetes 集群，其次需要安装 Helm，确保 helm 命令可以正常运行。
+
+## 安装
+
+安装直接使用 helm 命令在本文件夹运行即可，使用 `-n` 可以制定 NameSpace。
+
+```shell
+helm install proxypool-app . -n scrape
+```
+
+其中 proxypool-app 就是应用的名字，可以任意取名，它会用作代理池 Deplyment 的名称。
+
+如果需要覆盖变量，可以修改 values.yaml 文件，执行如下命令安装：
+
+```shell
+helm install proxypool-app . -f values.yaml -n scrape
+```
+
+## 更新
+
+如果需要更新配置，可以修改 values.yaml 文件，执行如下命令更新版本：
+
+```shell
+helm upgrade proxypool-app . -f values.yaml -n scrape
+```
+
+## 卸载
+
+如果不想使用了，可以只用 uninstall 命令卸载：
+
+```shell
+helm uninstall proxypool-app -n scrape
+```

kubernetes/templates/_helpers.tpl

@@ -0,0 +1,53 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "proxypool.name" -}}
+{{- default .Chart.Name .Values.name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "proxypool.fullname" -}}
+{{- if .Values.fullname }}
+{{- .Values.fullname | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.name }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "proxypool.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "proxypool.labels" -}}
+helm.sh/chart: {{ include "proxypool.chart" . }}
+{{ include "proxypool.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "proxypool.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "proxypool.fullname" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

kubernetes/templates/proxypool-deployment.yaml

@@ -0,0 +1,29 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "proxypool.fullname" . }}
+  labels:
+    {{- include "proxypool.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.deployment.replicas }}
+  revisionHistoryLimit: {{ .Values.deployment.revisionHistoryLimit }}
+  selector:
+    matchLabels:
+      {{- include "proxypool.labels" . | nindent 8 }}
+  template:
+    metadata:
+      labels:
+        {{- include "proxypool.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: {{ .Values.deployment.restartPolicy }}
+      containers:
+        - name: {{ include "proxypool.fullname" . }}
+          image: {{ .Values.deployment.image }}
+          ports:
+            - containerPort: 5555
+              protocol: TCP
+          imagePullPolicy: {{ .Values.deployment.imagePullPolicy }}
+          resources:
+            {{- toYaml .Values.deployment.resources | nindent 12 }}
+          env:
+            {{- toYaml .Values.deployment.env | nindent 12 }}

kubernetes/templates/proxypool-ingress.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "proxypool.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "proxypool.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ . }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+          {{- end }}
+    {{- end }}
+  {{- end }}

kubernetes/templates/proxypool-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "proxypool.fullname" . }}
+  labels:
+    {{- include "proxypool.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: 5555
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "proxypool.selectorLabels" . | nindent 4 }}

kubernetes/templates/redis-deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: proxypool-redis
+  name: proxypool-redis
+spec:
+  replicas: 1
+  revisionHistoryLimit: 1
+  selector:
+    matchLabels:
+      app: proxypool-redis
+  template:
+    metadata:
+      labels:
+        app: proxypool-redis
+    spec:
+      containers:
+        - image: redis:alpine
+          name: proxypool-redis
+          ports:
+            - containerPort: 6379
+          resources:
+            limits:
+              memory: "100Mi"
+              cpu: "100m"
+            requests:
+              memory: "100Mi"
+              cpu: "100m"
+      restartPolicy: Always

kubernetes/templates/redis-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: proxypool-redis
+  name: proxypool-redis
+spec:
+  ports:
+    - name: "6379"
+      port: 6379
+      targetPort: 6379
+  selector:
+    app: proxypool-redis

kubernetes/values.yaml

@@ -0,0 +1,37 @@
+name: proxypool
+fullname: proxypool-app
+
+deployment:
+  image: germey/proxypool:master
+  imagePullPolicy: Always
+  restartPolicy: Always
+  revisionHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  replicas: 1
+  resources:
+    limits:
+      memory: "200Mi"
+      cpu: "80m"
+    requests:
+      memory: "200Mi"
+      cpu: "80m"
+  env:
+    - name: REDIS_HOST
+      value: "proxypool-redis"
+
+service:
+  type: ClusterIP
+  port: 80
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx
+  hosts:
+    - host: proxypool.scrape.center
+      paths:
+        - "/"
+  tls:
+    - secretName: tls-wildcard-scrape-center
+      hosts:
+        - proxypool.scrape.center