feat: Windows support

It looks like for Windows in the production code: - sqlite3_initialize must be called explicitly to avoid access violation writing 0x0000000000000000 - using SQLITE_TRANSIENT as -1 causes access violation writing 0x00000000FFFFFFFF. This is probably since Python cases -1 to c_int, when it should be a pointer, and so c_void_p. Maybe in a later change could change it to SQLITE_STATIC, but not doing that now to keep this change small. And in tests: - to allow the test SQLite database to be deleted, the cursor and connection objects have to be removed from scope. Otherwise Windows thinks the file object is still open, and refuses to allow the deletion of the database at the end of the test Closes: #46
michalc · Jul 31, 2023 · c8a270f · c8a270f
1 parent 8a88357
commit c8a270f
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 17 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -9,21 +9,22 @@ jobs:
   download-coverage-reporter:
     runs-on: ubuntu-latest
     steps:
-      - name: "Download coverage reporter"
+      - name: "Download coverage reporters"
         run: |
           mkdir -p ./reporter
-          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./reporter/cc-test-reporter
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./reporter/cc-test-reporter-linux
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-windows-amd64 > ./reporter/cc-test-reporter-windows.exe
       - name: "Notify code climate of pending coverage upload"
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         run: |
-          chmod +x ./reporter/cc-test-reporter
-          ./reporter/cc-test-reporter before-build
-      - name: "Save reporter"
+          chmod +x ./reporter/cc-test-reporter-linux
+          ./reporter/cc-test-reporter-linux before-build
+      - name: "Save reporters"
         uses: actions/upload-artifact@v3
         with:
           name: reporter
-          path: ./reporter/cc-test-reporter
+          path: ./reporter/*
 
   # We want older SQLite amalgamation files, but they are not available to download,
   # so must be built from source. And they cannot be build on Windows, even for tests
@@ -69,6 +70,12 @@ jobs:
           - {os: "ubuntu-20.04", python: "3.9.0"}
           - {os: "ubuntu-20.04", python: "3.10.0"}
           - {os: "ubuntu-20.04", python: "3.11.0"}
+          - {os: "windows-2019", python: "3.6.7"}
+          - {os: "windows-2019", python: "3.7.1"}
+          - {os: "windows-2019", python: "3.8.0"}
+          - {os: "windows-2019", python: "3.9.0"}
+          - {os: "windows-2019", python: "3.10.0"}
+          - {os: "windows-2019", python: "3.11.0"}
     runs-on: '${{ matrix.os-and-python-version.os }}'
     env:
       MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE
@@ -90,16 +97,36 @@ jobs:
         with:
           name: sqlite-${{ matrix.sqlite-version }}
           path: .
-      - name: "Compile SQLite from amalgamation"
-        if: matrix.sqlite-version != 'default'
+      - name: "Compile SQLite from amalgamation (Windows)"
+        if: matrix.os-and-python-version.os == 'windows-2019' && matrix.sqlite-version != 'default'
+        run: |
+          gcc -shared sqlite3.c -o sqlite3.dll
+          echo "SQLITE3_VERSION=${{ matrix.sqlite-version }}" >> $env:GITHUB_ENV
+          echo "LIBSQLITE3_PATH=${PWD}/sqlite3.dll" >> $env:GITHUB_ENV
+      - name: "Compile SQLite from amalgamation (Ubuntu)"
+        if: matrix.os-and-python-version.os == 'ubuntu-20.04' && matrix.sqlite-version != 'default'
         run: |
           gcc -shared -fPIC -o libsqlite3.so.0 sqlite3.c
           echo "SQLITE3_VERSION=${{ matrix.sqlite-version }}" >> "$GITHUB_ENV"
           echo "LIBSQLITE3_PATH=${PWD}/libsqlite3.so.0" >> "$GITHUB_ENV"
       - name: "Install sqlite-s3-query and any dependencies"
         run: |
           pip install ".[dev]"
-      - name: "Test"
+      - name: "Test (Windows)"
+        if: matrix.os-and-python-version.os == 'windows-2019'
+        run: |
+          Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2023-07-21T21-12-44Z" -OutFile "./minio.exe"
+          mkdir -p ./data
+          ./minio.exe server ./data &
+          do {
+            Write-Host "Waiting for MinIO"
+            sleep 3
+          } until(Test-NetConnection 127.0.0.1 -Port 9000 | ? { $_.TcpTestSucceeded } )
+          coverage run -m unittest
+          coverage xml
+          ./reporter/cc-test-reporter-windows.exe format-coverage --output "./coverage/${{ matrix.os-and-python-version.os }}-${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json"
+      - name: "Test (Ubuntu)"
+        if: matrix.os-and-python-version.os == 'ubuntu-20.04'
         run: |
           wget -O minio https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-07-21T21-12-44Z
           chmod +x minio
@@ -108,9 +135,9 @@ jobs:
           timeout 22 sh -c 'until nc -z $0 $1; do sleep 1; done' 127.0.0.1 9000
           coverage run -m unittest
           coverage xml
-          chmod +x ./reporter/cc-test-reporter
-          COVERAGE_FILE_NAME="./coverage/${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json"
-          ./reporter/cc-test-reporter format-coverage --output "$COVERAGE_FILE_NAME"
+          chmod +x ./reporter/cc-test-reporter-linux
+          COVERAGE_FILE_NAME="./coverage/${{ matrix.os-and-python-version.os }}-${{ matrix.os-and-python-version.python }}-${{ matrix.sqlite-version }}.json"
+          ./reporter/cc-test-reporter-linux format-coverage --output "$COVERAGE_FILE_NAME"
       - name: "Save code coverage"
         uses: actions/upload-artifact@v3
         with:
@@ -127,6 +154,6 @@ jobs:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         run: |
           ls -R
-          chmod +x ./reporter/cc-test-reporter
-          ./reporter/cc-test-reporter sum-coverage ./coverage/*.json -p 24
-          ./reporter/cc-test-reporter upload-coverage
+          chmod +x ./reporter/cc-test-reporter-linux
+          ./reporter/cc-test-reporter-linux sum-coverage ./coverage/*.json -p 48
+          ./reporter/cc-test-reporter-linux upload-coverage
diff --git a/sqlite_s3_query.py b/sqlite_s3_query.py
@@ -35,7 +35,7 @@ def sqlite_s3_query_multi(url, get_credentials=lambda now: (
     SQLITE_NOTFOUND = 12
     SQLITE_ROW = 100
     SQLITE_DONE = 101
-    SQLITE_TRANSIENT = -1
+    SQLITE_TRANSIENT = c_void_p(-1)
     SQLITE_OPEN_READONLY = 0x00000001
     SQLITE_OPEN_NOMUTEX = 0x00008000
     SQLITE_IOCAP_IMMUTABLE = 0x00002000
@@ -62,6 +62,8 @@ def sqlite_s3_query_multi(url, get_credentials=lambda now: (
         5: lambda pp_stmt, i: None,
     }
 
+    libsqlite3.sqlite3_initialize()
+
     vfs_name = b's3-' + str(uuid4()).encode()
     file_name = b's3-' + str(uuid4()).encode()
     body_hash = sha256(b'').hexdigest()

diff --git a/test.py b/test.py
@@ -690,7 +690,7 @@ def stream(self, method, url, headers, params):
             put_object_with_versioning('my-bucket', 'my.db', db)
 
         with server() as server_sock:
-            with self.assertRaisesRegex(Exception, 'Server disconnected|Connection reset'):
+            with self.assertRaisesRegex(Exception, 'Server disconnected|Connection reset|WinError 10053|WinError 10054'):
                 sqlite_s3_query('http://localhost:9000/my-bucket/my.db', get_credentials=lambda now: (
                     'us-east-1',
                     'AKIAIOSFODNN7EXAMPLE',
@@ -838,6 +838,9 @@ def get_db(sqls):
                 cur.execute(sql, params)
             cur.execute('COMMIT')
 
+        # Really close the file, especially on Windows
+        del cur, con
+
         def db():
             with open(db_path, 'rb') as f:
                 while True: