From c8a270f4678acace0cfa8c0e31ecde7856a0e9b5 Mon Sep 17 00:00:00 2001 From: Michal Charemza Date: Sun, 30 Jul 2023 10:55:07 +0100 Subject: [PATCH] feat: Windows support It looks like for Windows in the production code: - sqlite3_initialize must be called explicitly to avoid access violation writing 0x0000000000000000 - using SQLITE_TRANSIENT as -1 causes access violation writing 0x00000000FFFFFFFF. This is probably since Python cases -1 to c_int, when it should be a pointer, and so c_void_p. Maybe in a later change could change it to SQLITE_STATIC, but not doing that now to keep this change small. And in tests: - to allow the test SQLite database to be deleted, the cursor and connection objects have to be removed from scope. Otherwise Windows thinks the file object is still open, and refuses to allow the deletion of the database at the end of the test Closes: https://github.com/michalc/sqlite-s3-query/issues/46 --- .github/workflows/test.yml | 57 ++++++++++++++++++++++++++++---------- sqlite_s3_query.py | 4 ++- test.py | 5 +++- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9a65ba..6abb25f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,21 +9,22 @@ jobs: download-coverage-reporter: runs-on: ubuntu-latest steps: - - name: "Download coverage reporter" + - name: "Download coverage reporters" run: | mkdir -p ./reporter - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./reporter/cc-test-reporter + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./reporter/cc-test-reporter-linux + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-windows-amd64 > ./reporter/cc-test-reporter-windows.exe - name: "Notify code climate of pending coverage upload" env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} run: | - chmod +x ./reporter/cc-test-reporter - ./reporter/cc-test-reporter before-build - - name: "Save reporter" + chmod +x ./reporter/cc-test-reporter-linux + ./reporter/cc-test-reporter-linux before-build + - name: "Save reporters" uses: actions/upload-artifact@v3 with: name: reporter - path: ./reporter/cc-test-reporter + path: ./reporter/* # We want older SQLite amalgamation files, but they are not available to download, # so must be built from source. And they cannot be build on Windows, even for tests @@ -69,6 +70,12 @@ jobs: - {os: "ubuntu-20.04", python: "3.9.0"} - {os: "ubuntu-20.04", python: "3.10.0"} - {os: "ubuntu-20.04", python: "3.11.0"} + - {os: "windows-2019", python: "3.6.7"} + - {os: "windows-2019", python: "3.7.1"} + - {os: "windows-2019", python: "3.8.0"} + - {os: "windows-2019", python: "3.9.0"} + - {os: "windows-2019", python: "3.10.0"} + - {os: "windows-2019", python: "3.11.0"} runs-on: '${{ matrix.os-and-python-version.os }}' env: MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE @@ -90,8 +97,14 @@ jobs: with: name: sqlite-${{ matrix.sqlite-version }} path: . - - name: "Compile SQLite from amalgamation" - if: matrix.sqlite-version != 'default' + - name: "Compile SQLite from amalgamation (Windows)" + if: matrix.os-and-python-version.os == 'windows-2019' && matrix.sqlite-version != 'default' + run: | + gcc -shared sqlite3.c -o sqlite3.dll + echo "SQLITE3_VERSION=${{ matrix.sqlite-version }}" >> $env:GITHUB_ENV + echo "LIBSQLITE3_PATH=${PWD}/sqlite3.dll" >> $env:GITHUB_ENV + - name: "Compile SQLite from amalgamation (Ubuntu)" + if: matrix.os-and-python-version.os == 'ubuntu-20.04' && matrix.sqlite-version != 'default' run: | gcc -shared -fPIC -o libsqlite3.so.0 sqlite3.c echo "SQLITE3_VERSION=${{ matrix.sqlite-version }}" >> "$GITHUB_ENV" @@ -99,7 +112,21 @@ jobs: - name: "Install sqlite-s3-query and any dependencies" run: | pip install ".[dev]" - - name: "Test" + - name: "Test (Windows)" + if: matrix.os-and-python-version.os == 'windows-2019' + run: | + Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2023-07-21T21-12-44Z" -OutFile "./minio.exe" + mkdir -p ./data + ./minio.exe server ./data & + do { + Write-Host "Waiting for MinIO" + sleep 3 + } until(Test-NetConnection 127.0.0.1 -Port 9000 | ? { $_.TcpTestSucceeded } ) + coverage run -m unittest + coverage xml + ./reporter/cc-test-reporter-windows.exe format-coverage --output "./coverage/${{ matrix.os-and-python-version.os }}-${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json" + - name: "Test (Ubuntu)" + if: matrix.os-and-python-version.os == 'ubuntu-20.04' run: | wget -O minio https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-07-21T21-12-44Z chmod +x minio @@ -108,9 +135,9 @@ jobs: timeout 22 sh -c 'until nc -z $0 $1; do sleep 1; done' 127.0.0.1 9000 coverage run -m unittest coverage xml - chmod +x ./reporter/cc-test-reporter - COVERAGE_FILE_NAME="./coverage/${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json" - ./reporter/cc-test-reporter format-coverage --output "$COVERAGE_FILE_NAME" + chmod +x ./reporter/cc-test-reporter-linux + COVERAGE_FILE_NAME="./coverage/${{ matrix.os-and-python-version.os }}-${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json" + ./reporter/cc-test-reporter-linux format-coverage --output "$COVERAGE_FILE_NAME" - name: "Save code coverage" uses: actions/upload-artifact@v3 with: @@ -127,6 +154,6 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} run: | ls -R - chmod +x ./reporter/cc-test-reporter - ./reporter/cc-test-reporter sum-coverage ./coverage/*.json -p 24 - ./reporter/cc-test-reporter upload-coverage + chmod +x ./reporter/cc-test-reporter-linux + ./reporter/cc-test-reporter-linux sum-coverage ./coverage/*.json -p 48 + ./reporter/cc-test-reporter-linux upload-coverage diff --git a/sqlite_s3_query.py b/sqlite_s3_query.py index 5c3ff40..7b9dad3 100644 --- a/sqlite_s3_query.py +++ b/sqlite_s3_query.py @@ -35,7 +35,7 @@ def sqlite_s3_query_multi(url, get_credentials=lambda now: ( SQLITE_NOTFOUND = 12 SQLITE_ROW = 100 SQLITE_DONE = 101 - SQLITE_TRANSIENT = -1 + SQLITE_TRANSIENT = c_void_p(-1) SQLITE_OPEN_READONLY = 0x00000001 SQLITE_OPEN_NOMUTEX = 0x00008000 SQLITE_IOCAP_IMMUTABLE = 0x00002000 @@ -62,6 +62,8 @@ def sqlite_s3_query_multi(url, get_credentials=lambda now: ( 5: lambda pp_stmt, i: None, } + libsqlite3.sqlite3_initialize() + vfs_name = b's3-' + str(uuid4()).encode() file_name = b's3-' + str(uuid4()).encode() body_hash = sha256(b'').hexdigest() diff --git a/test.py b/test.py index cbcc96f..d79a8f2 100644 --- a/test.py +++ b/test.py @@ -690,7 +690,7 @@ def stream(self, method, url, headers, params): put_object_with_versioning('my-bucket', 'my.db', db) with server() as server_sock: - with self.assertRaisesRegex(Exception, 'Server disconnected|Connection reset'): + with self.assertRaisesRegex(Exception, 'Server disconnected|Connection reset|WinError 10053|WinError 10054'): sqlite_s3_query('http://localhost:9000/my-bucket/my.db', get_credentials=lambda now: ( 'us-east-1', 'AKIAIOSFODNN7EXAMPLE', @@ -838,6 +838,9 @@ def get_db(sqls): cur.execute(sql, params) cur.execute('COMMIT') + # Really close the file, especially on Windows + del cur, con + def db(): with open(db_path, 'rb') as f: while True: