[go: nahoru, domu]
Skip to content

Commit

Permalink
Re-using scritpts from safetensors. (#1328)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Narsil
Narsil committed Aug 23, 2023
1 parent d0bb35d commit 6c350d8
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 48 deletions.
128 changes: 80 additions & 48 deletions .github/workflows/python-release.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
name: Python Release

on:
push:
tags:
Expand All @@ -13,13 +12,21 @@ env:
DIST_DIR: ${{ github.sha }}

jobs:
lock_exists:
runs-on: ubuntu-latest
name: Cargo.lock
steps:
- uses: actions/checkout@v3
- name: Cargo.lock lock exists
run: cat Cargo.lock
working-directory: ./bindings/python

create_wheels_manylinux:
runs-on: ubuntu-latest
needs: [lock_exists]
name: Create wheels for manylinux2014
container: quay.io/pypa/manylinux2014_x86_64
steps:
# v1 is required when using manylinux2010
- uses: actions/checkout@v3

- name: Install dependencies
Expand All @@ -29,37 +36,48 @@ jobs:
working-directory: ./bindings/python
run: sh build-wheels.sh

create_wheels_windows_32bit:
name: Create wheels for windows 32-bit
create_wheels_windows:
name: Windows
runs-on: windows-latest
needs: [lock_exists]
strategy:
matrix:
python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
bits: ["32", "64"]
steps:
- name: Checkout repository
uses: actions/checkout@v2

- name: Install Rust
- name: Install Rust 32bits
if: ${{ matrix.os == '32' }}
uses: actions-rs/toolchain@v1
with:
toolchain: stable-i686-pc-windows-msvc
override: true

- name: Install Rust 64bits
if: ${{ matrix.os == '32' }}
uses: actions-rs/toolchain@v1
with:
toolchain: stable-i686-pc-windows-msvc
override: true

- name: Override toolchain
if: ${{ matrix.os == '32' }}
shell: bash
working-directory: ./bindings/python
run: echo "stable-i686-pc-windows-msvc" > rust-toolchain

- name: Install Python
uses: actions/setup-python@v1
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
architecture: x86

- name: Install dependencies
run: |
# On old versions of python there is an old version of setuptools already installed
pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
- name: Build wheel
working-directory: ./bindings/python
Expand All @@ -71,59 +89,67 @@ jobs:
pip install awscli
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
create_wheels_others_64bit:
name: Other OSes
env:
MACOSX_DEPLOYMENT_TARGET: 10.11
create_wheels_macos_conda:
name: MacOS - Conda
runs-on: ${{ matrix.os }}
needs: [lock_exists]
strategy:
matrix:
os: [windows-latest, macos-latest]
python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
os: [macos-latest]
# 3.11 not available on Conda yet.
python: ["3.7", "3.8", "3.9", "3.10"]

steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Install miniconda
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
python-version: ${{ matrix.python }}

- name: Conda info
shell: bash -l {0}
run: conda info

- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true

- name: Install Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
architecture: x64

- name: Install dependencies
- name: Setup conda env
shell: bash -l {0}
run: |
# On old versions of python there is an old version of setuptools already installed
pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
conda install setuptools-rust
conda install -c defaults anaconda-client conda-build
- name: Build wheel
- name: Extract version
shell: bash -l {0}
working-directory: ./bindings/python
run: python setup.py bdist_wheel
run: echo "TOKENIZERS_VERSION=`python setup.py --version`" >> $GITHUB_ENV

- name: Rename wheels
shell: bash
working-directory: ./bindings/python/dist
run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
- name: Build conda packages
shell: bash -l {0}
working-directory: ./bindings/python
run: |
MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel
- name: Upload wheels
shell: bash
run: |
pip install awscli
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
create_wheels_macos_13:
name: MacOS 13
runs-on: ${{ matrix.os }}
create_wheels_macos:
name: MacOS
runs-on: ${{ matrix.os.os }}
needs: [lock_exists]
strategy:
matrix:
os: [macos-13]
python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
# os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}]
os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}]
steps:
- name: Checkout repository
uses: actions/checkout@v2
Expand All @@ -138,33 +164,36 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
architecture: x64

- name: Install dependencies
run: |
# On old versions of python there is an old version of setuptools already installed
pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
- name: Build wheel
pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
- name: Override target
if: ${{ matrix.os.target }}
working-directory: ./bindings/python
run: python setup.py bdist_wheel
run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV

- name: Rename wheels
shell: bash
working-directory: ./bindings/python/dist
run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
- name: Build wheel
working-directory: ./bindings/python
run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel

- name: Upload wheels
shell: bash
run: |
pip install awscli
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
create_wheels_macos_arm64:
name: Create wheels for MacOS M1
name: MacOS M1
runs-on: macos-arm64
needs: [lock_exists]
strategy:
matrix:
python: [ "3.9.13", "3.10.6", "3.11.1"]
python: ["3.9.13", "3.10.6", "3.11.0"]
# target: ["12.0", "13.0", "14.0"]
target: ["12.0", "13.0"]
steps:
- name: Checkout repository
Expand Down Expand Up @@ -192,16 +221,19 @@ jobs:
cd ../../
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
upload_package:
Upload_package:
name: Upload package to PyPi
runs-on: ubuntu-latest
needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13]
needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda]

steps:
- uses: actions/checkout@v3

- name: Install Python
uses: actions/setup-python@v1
uses: actions/setup-python@v4
with:
python-version: "3.10"
architecture: x64

- name: Retrieve all wheels
shell: bash
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/rust-release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,6 @@ jobs:

- name: Publish package rust
working-directory: ./tokenizers
if: ${{ !contains(github.ref, 'rc') }}
run: cargo publish --token ${CRATES_TOKEN}

0 comments on commit 6c350d8

Please sign in to comment.