update python to 3.9

2024-11-21 01:16:33 +08:00 · 2023-04-06 22:48:00 +08:00 · 2023-04-06 22:48:00 +08:00 · aa6416a229
commit aa6416a229
parent 180407622a
7 changed files with 119 additions and 109 deletions
--- a/install/docker/Dockerfile
+++ b/install/docker/Dockerfile
@ -1,27 +1,17 @@
-# docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01 -f install/docker/Dockerfile .
+# docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01 -f install/docker/Dockerfile .

-FROM ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:base
+FROM ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:base-python3.9

-RUN apt-get update && apt-get install -y -f mysql-client zip git
-
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash
-RUN apt-get -y install nodejs && npm install yarn -g
-
-RUN rm /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python && rm /usr/bin/pip && ln -s /usr/bin/pip3 /usr/bin/pip
-RUN pip install pip --upgrade && pip install docstring-parser==0.14.1 kfp==1.8.0
-RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/bin/
-RUN mkdir /kubeflow && wget -P /kubeflow/ https://docker-76009.sz.gfp.tencent-cloud.com/kubeflow/src.tar.gz && tar -zxvf /kubeflow/src.tar.gz -C /kubeflow/ && rm /kubeflow/src.tar.gz
-
-RUN pip install kubernetes==18.20.0 click==6.7 jaeger-client==4.6.1 opentracing-instrumentation==3.3.1
-COPY install/docker/docker-add-file/rest.py /usr/local/lib/python3.6/dist-packages/kubernetes/client/rest.py
-COPY install/docker/docker-add-file/__init__.py /usr/local/lib/python3.6/dist-packages/marshmallow_enum/__init__.py
+COPY install/docker/docker-add-file/rest.py /usr/local/lib/python3.9/dist-packages/kubernetes/client/rest.py
+COPY install/docker/docker-add-file/__init__.py /usr/local/lib/python3.9/dist-packages/marshmallow_enum/__init__.py

+RUN pip3 install Pillow
 COPY myapp /home/myapp/myapp
 COPY myapp/static/appbuilder/frontend /data/web/frontend
+COPY aihub /cube-studio/aihub
 ENV PATH=/home/myapp/myapp/bin:$PATH
-ENV PYTHONPATH=/home/myapp:/kubeflow/src/pipelines/sdk/python/:$PYTHONPATH
+ENV PYTHONPATH=/home/myapp:$PYTHONPATH

-USER root
 COPY install/docker/entrypoint.sh /entrypoint.sh

 # ENTRYPOINT ["/entrypoint.sh"]
@ -31,4 +21,3 @@ COPY install/docker/entrypoint.sh /entrypoint.sh



-
--- a/install/docker/Dockerfile-base
+++ b/install/docker/Dockerfile-base
@ -1,40 +1,48 @@
-FROM python:3.6-jessie
+# docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:base-python3.9 -f install/docker/Dockerfile-base .

-# Configure environment
-ENV LANG=C.UTF-8 \
-    LC_ALL=C.UTF-8
+FROM ubuntu:20.04

-RUN apt-get update -y && \
-    apt-get install -y apt-transport-https apt-utils build-essential libssl-dev \
-    libglib2.0-0 libnss3 libgconf-2-4 libfontconfig1 \
-    vim less postgresql-client redis-tools \
-    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev && \
-    curl -sL https://deb.nodesource.com/setup_10.x | bash -
+ENV TZ=Asia/Shanghai
+ENV DEBIAN_FRONTEND=noninteractive
+ARG PYTHON_VERSION=python3.9

+# 替换源
+COPY install/docker/sources.list /etc/apt/sources.list
+RUN apt-get clean all && apt-get update -y

-RUN apt-get install -y --force-yes nodejs && \
-    apt-get install -y --force-yes --no-install-recommends ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy
+# 安装运维工具
+RUN apt install -y --force-yes --no-install-recommends apt-utils ca-certificates software-properties-common vim apt-transport-https gnupg2 ca-certificates-java rsync jq  wget git dnsutils iputils-ping net-tools curl mysql-client locales zip

-RUN pip install clickhouse-sqlalchemy pysnooper pymysql gevent mysqlclient flower infi.clickhouse_orm opencv-python numpy sqlalchemy_utils
+# 安装python
+RUN add-apt-repository -y ppa:deadsnakes/ppa && apt update && apt install -y  libsasl2-dev libpq-dev python3-pip python3.6-distutils
+RUN set -x; rm -rf /usr/bin/python /usr/bin/python3; apt install -y --fix-missing ${PYTHON_VERSION} ${PYTHON_VERSION}-dev && ln -s /usr/bin/${PYTHON_VERSION} /usr/bin/python && ln -s /usr/bin/${PYTHON_VERSION} /usr/bin/python3 && rm -rf /usr/bin/pip && ln -s /usr/bin/pip3 /usr/bin/pip && pip3 install --upgrade pip setuptools wheel

-WORKDIR /home/myapp
+# 安装node
+RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash
+RUN apt-get install -y nodejs && npm install yarn -g

-COPY install/docker/requirements.txt .
-COPY install/docker/requirements-dev.txt .
+# 安装字体
+RUN apt install -y --force-yes --no-install-recommends locales ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy && locale-gen zh_CN && locale-gen zh_CN.utf8
+ENV LANG zh_CN.UTF-8
+ENV LC_ALL zh_CN.UTF-8
+ENV LANGUAGE zh_CN.UTF-8
+
+# 安装kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/bin/
+
+# 便捷操作
+RUN echo "alias ll='ls -alF'" >> /root/.bashrc && \
+    echo "alias la='ls -A'" >> /root/.bashrc && \
+    echo "alias vi='vim'" >> /root/.bashrc && \
+    /bin/bash -c "source /root/.bashrc"
+
+COPY install/docker/requirements.txt /requirements.txt

 RUN pip install --upgrade setuptools pip && \
-    pip install -r requirements.txt -r requirements-dev.txt && \
-    rm -rf /root/.cache/pip && \
-    CHROMEDRIVER_VERSION=`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE` && \
-    mkdir -p /opt/chromedriver-$CHROMEDRIVER_VERSION && \
-    curl -sS -o /tmp/chromedriver_linux64.zip http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip && \
-    unzip -qq /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \
-    rm /tmp/chromedriver_linux64.zip && \
-    chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \
-    ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver && \
-    wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
-    dpkg -i google-chrome-stable_current_amd64.deb; apt-get -fy install && \
-    rm google-chrome-stable_current_amd64.deb
+    pip install -r /requirements.txt && \
+    rm -rf /root/.cache/pip
+
+WORKDIR /home/myapp

 USER root

--- a/install/docker/README.md
+++ b/install/docker/README.md
@ -5,9 +5,9 @@

 ```
 linux
-docker run --network host --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -v $PWD/mysql:/var/lib/mysql -v $PWD/docker-add-file/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf -d mysql:5.7
+docker run --network host --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -e MYSQL_ALLOW_EMPTY_PASSWORD=true -v $PWD/docker-add-file/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf -d mysql:5.7
 mac
-docker run -p 3306:3306 --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -v $PWD/mysql:/var/lib/mysql -d mysql:5.7
+docker run -p 3306:3306 --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -d mysql:5.7

 ```
 进入mysql，创建kubeflow数据库
@ -27,16 +27,16 @@ mysql> flush privileges;
 docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:base -f install/docker/Dockerfile-base .

 使用基础镜像构建生产镜像
-docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01 -f install/docker/Dockerfile .
+docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01 -f install/docker/Dockerfile .

 构建frontend镜像
-docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:frontend-2022.09.01 -f install/docker/dockerFrontend/Dockerfile .
+docker build -t ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard-frontend:2023.04.01 -f install/docker/dockerFrontend/Dockerfile .
 ```

 ## 镜像拉取(如果你不参与开发可以直接使用线上镜像)
 ```
-docker pull ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01
-docker pull ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:frontend-2022.09.01
+docker pull ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01
+docker pull ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard-frontend:2023.04.01
 ```

 ## deploy myapp (docker-compose)
@ -155,4 +155,4 @@ npm config set registry https://registry.npmmirror.com
 yarn配置镜像命令
 ```
 yarn config set registry https://registry.npmmirror.com
-```
+```
--- a/install/docker/docker-compose.yml
+++ b/install/docker/docker-compose.yml
@ -19,9 +19,10 @@ services:
 #      - "3306:3306"
 #    volumes:
 #      - ./data/mysql:/var/lib/mysql
+#      - ./docker-add-file/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf

  frontend:
-    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard-frontend:2022.12.01
+    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard-frontend:2023.04.01
    restart: unless-stopped
    ports:
      - '80:80'
@ -31,8 +32,8 @@ services:
      - ../../install/docker/dockerFrontend/nginx.80.conf:/etc/nginx/conf.d/default.conf

  myapp:
-    image:  ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01
-    restart: always
+    image:  ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01
+    restart: unless-stopped
    command: ['bash','-c','/entrypoint.sh']
    environment:
      STAGE: 'dev'
@ -50,16 +51,17 @@ services:
    volumes:
      - ../../myapp/static/appbuilder/frontend:/data/web/frontend
      - ../../myapp/:/home/myapp/myapp/
-      - ./file:/pvc
+      - ../../aihub/:/cube-studio/aihub/
+      - ./file:/data/k8s/kubeflow
      - ./entrypoint.sh:/entrypoint.sh
      - ./config.py:/home/myapp/myapp/config.py
      - ./kubeconfig:/home/myapp/kubeconfig

-
+#
 #  beat:
-#    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01
+#    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01
 #    restart: "no"
-#    command: ["bash","-c","celery beat --app=myapp.tasks.celery_app:celery_app --loglevel=info"]
+#    command: ["bash","-c","celery --app=myapp.tasks.celery_app:celery_app beat --loglevel=info"]
 #    shm_size: '100gb'
 #    environment:
 #      REDIS_HOST: 'redis'
@ -77,12 +79,11 @@ services:
 #      - ./kubeconfig:/home/myapp/kubeconfig
 ##
 #
-##
-#
 #  worker:
-#    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2022.12.01
+#    image: ccr.ccs.tencentyun.com/cube-studio/kubeflow-dashboard:2023.04.01
 #    restart: unless-stopped
-#    command: ["bash","-c","celery worker --app=myapp.tasks.celery_app:celery_app --loglevel=info --pool=prefork -Ofair -c 40"]
+#    # 默认的prefork池针对长时间的任务支持不是很好，如果任务运行时间有数分钟/小时，建议启用 Celery 的 -Ofair 命令参数
+#    command: ["bash","-c","celery --app=myapp.tasks.celery_app:celery_app worker --loglevel=info --pool=prefork -Ofair -c 2 -n worker@%h"]
 #    environment:
 #      REDIS_HOST: 'redis'
 #      REDIS_PORT: '6379'
@ -93,10 +94,10 @@ services:
 #      - redis
 #    volumes:
 #      - ../../myapp/:/home/myapp/myapp/
-#      - ./file:/pvc
+#      - ./file:/data/k8s/kubeflow
 #      - ./entrypoint.sh:/entrypoint.sh
 #      - ./config.py:/home/myapp/myapp/config.py
 #      - ./kubeconfig:/home/myapp/kubeconfig
-#
+


--- a/install/docker/entrypoint.sh
+++ b/install/docker/entrypoint.sh
@ -4,8 +4,14 @@ set -ex

 rm -rf /home/myapp/myapp/static/assets
 ln -s /home/myapp/myapp/assets /home/myapp/myapp/static/
-rm -rf /home/myapp/myapp/static/mnt
+rm -f /home/myapp/myapp/static/mnt
+mkdir -p /data/k8s/kubeflow/pipeline/workspace
 ln -s /data/k8s/kubeflow/pipeline/workspace /home/myapp/myapp/static/mnt
+rm -f /home/myapp/myapp/static/dataset
+mkdir -p /data/k8s/kubeflow/dataset
+ln -s /data/k8s/kubeflow/dataset /home/myapp/myapp/static/
+rm -f /home/myapp/myapp/static/aihub
+ln -s /cube-studio/aihub /home/myapp/myapp/static/

 export FLASK_APP=myapp:app
 python myapp/create_db.py
@ -19,6 +25,7 @@ myapp init
 if [ "$STAGE" = "build" ]; then
 #  cd /home/myapp/myapp/vision && yarn && yarn build
  cd /home/myapp/myapp/vision && npm install && yarn build
+  cd /home/myapp/myapp/frontend && npm install && yarn build

 elif [ "$STAGE" = "dev" ]; then
  export FLASK_APP=myapp:app
--- a/install/docker/requirements-dev.txt
+++ b/install/docker/requirements-dev.txt
@ -1,19 +0,0 @@
-black==19.3b0
-coverage==4.5.3
-flake8-import-order==0.18.1
-flake8-mypy==17.8.0
-flake8==3.7.7
-flask-cors==3.0.7
-ipdb==0.12
-mypy==0.670
-nose==1.3.7
-pip-tools==3.7.0
-pre-commit==1.17.0
-psycopg2-binary==2.8
-pycodestyle==2.5.0
-pylint==1.9.2
-redis==3.2.1
-requests==2.22.0
-statsd==3.3.0
-tox==3.11.1
-bs4==0.0.1
--- a/install/docker/requirements.txt
+++ b/install/docker/requirements.txt
@ -1,30 +1,41 @@
 alembic==1.0.11           # via flask-migrate
-amqp==2.5.0               # via kombu
+amqp==5.1.1               # via kombu
 apispec[yaml]==1.3.3      # via flask-appbuilder
 asn1crypto==0.24.0        # via cryptography
 attrs==19.1.0             # via jsonschema
 babel==2.7.0              # via flask-babel
-billiard==3.6.0.0         # via celery
+billiard==3.6.4.0         # via celery
+black==19.3b0
 bleach==3.1.0
-celery==4.3.0
+bs4==0.0.1
+celery==5.1.2
 certifi==2019.6.16        # via requests
-cffi==1.12.3              # via cryptography
+cffi==1.15.1              # via cryptography
 chardet==3.0.4            # via requests
-click==6.7
+click==7.1.2
 colorama==0.4.1
 contextlib2==0.5.5
+cos-python-sdk-v5==1.9.22
+coverage==4.5.3
 croniter==0.3.30
 cryptography==2.7
 decorator==4.4.0          # via retry
 defusedxml==0.6.0         # via python3-openid
+docstring-parser==0.14.1
+flake8-import-order==0.18.1
+flake8-mypy==17.8.0
+flake8==3.7.7
 flask-appbuilder==2.2.2   # 2.2.2   2.1.7
 flask-babel==0.12.2       # via flask-appbuilder
 flask-caching==1.7.2
 flask-compress==1.4.0
+flask-cors==3.0.7
 flask-jwt-extended==3.20.0  # via flask-appbuilder
 flask-login==0.4.1        # via flask-appbuilder
 flask-migrate==2.5.2
-flask-openid==1.2.5       # via flask-appbuilder
+flask-openid==1.3.0       # via flask-appbuilder
+Flask-SocketIO==5.3.2
+Flask-Sockets==0.2.1
 flask-sqlalchemy==2.4.0   # via flask-appbuilder, flask-migrate
 flask-talisman==0.7.0
 flask-wtf==0.14.2
@ -32,37 +43,58 @@ flask==1.1.1
 future==0.17.1            # via parsedatetime
 geographiclib==1.49       # via geopy
 geopy==1.20.0
+gevent-websocket==0.10.1
+gevent==22.10.2
 gunicorn==19.8.1
+html-table==1.0
 humanize==0.5.1
 idna==2.8
+ipdb==0.13.11
 isodate==0.6.0
 itsdangerous==1.1.0       # via flask
+jaeger-client==4.6.1
 jinja2==2.10.1            # via flask, flask-babel
 jsonschema==3.0.1         # via flask-appbuilder
-kombu==4.6.3              # via celery
+kombu==5.1.0              # via celery
+kubernetes==18.20.0
 mako==1.0.14              # via alembic
 markdown==3.1.1
 markupsafe==1.1.1         # via jinja2, mako
 marshmallow-enum==1.4.1   # via flask-appbuilder
 marshmallow-sqlalchemy==0.17.0  # via flask-appbuilder
 marshmallow==2.19.5       # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
-numpy==1.17.0             # via pandas
-pandas==0.24.2
+minio==7.1.13
+mypy==1.1.1    # 不成功
+nose==1.3.7
+numpy==1.24.2             # via pandas
+opentracing-instrumentation==3.3.1
+pandas==1.5.3
 parsedatetime==2.4
 pathlib2==2.3.4
+pip-tools==3.7.0
 polyline==1.4.0
+pre-commit==1.17.0
 prison==0.1.2             # via flask-appbuilder
+psycopg2-binary==2.8
 py==1.8.0                 # via retry
+pyarrow==6.0.1
+pycodestyle==2.5.0
 pycparser==2.19           # via cffi
 pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
+pylint==1.9.2
+pymysql==1.0.2
 pyrsistent==0.15.4        # via jsonschema
-python-dateutil==2.8.0
+PySnooper==0.4.2
+python-dateutil==2.8.1
 python-dotenv==0.10.3
 python-editor==1.0.4      # via alembic
 python-geohash==0.8.5
 python3-openid==3.1.0     # via flask-openid
-pytz==2019.2              # via babel, celery, pandas
+pytz==2022.7.1              # via babel, celery, pandas
 pyyaml==5.4.1
+qrcode==7.3.1
+redis==3.2.1
+requests==2.22.0
 retry==0.9.2
 selenium==3.141.0
 simplejson==3.16.0
@ -70,19 +102,11 @@ six==1.12.0               # via bleach, cryptography, flask-jwt-extended, flask-
 sqlalchemy-utils==0.34.1
 sqlalchemy==1.3.6
 sqlparse==0.2.4
+statsd==3.3.0
+tox==3.11.1
 urllib3==1.25.3           # via requests, selenium
-vine==1.3.0               # via amqp, celery
+vine==5.0.0               # via amqp, celery
 webencodings==0.5.1       # via bleach
 werkzeug==0.15.5          # via flask, flask-jwt-extended
 wtforms-json==0.3.3
-wtforms==2.2.1            # via flask-wtf, wtforms-json
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.0.1        # via jsonschema, markdown
-
-docstring-parser==0.14.1
-kfp==1.8.8
-kubernetes==18.20.0
-click==7.1.2
-jaeger-client==4.6.1
-opentracing-instrumentation==3.3.1
+wtforms==2.2.1            # via flask-wtf, wtforms-json