[codex] Stabilize Harbor-backed test runs

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - "1.6"
+          - "1.10"
           - 1 # automatically expands to the latest stable 1.x release of Julia
           - nightly
         os:
@@ -21,7 +21,7 @@ jobs:
           - x64
     steps:
       - uses: actions/checkout@v5
-      - run: docker compose up -d
+      - run: docker info
       - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
@@ -50,4 +50,4 @@ jobs:
       - run: julia --project=docs docs/make.jl
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}

Project.toml

@@ -17,14 +17,17 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 [compat]
 DBInterface = "2.5"
 DecFP = "0.4.9, 0.4.10, 1"
+Harbor = "1.0.3"
 MariaDB_Connector_C_jll = "3.1.12"
 OpenSSL_jll = "3"
 Parsers = "0.3, 1, 2"
 Tables = "1"
-julia = "1.6"
+julia = "1.10"
 
 [extras]
+Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376"
+Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Harbor", "Sockets", "Test"]

README.md

@@ -16,12 +16,10 @@ Package for interfacing with MySQL databases from Julia via the MariaDB C connec
 [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://mysql.juliadatabases.org/stable)
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://mysql.juliadatabases.org/dev)
 
-## Contributing
-
-The tests require a MySQL DB to be running, which is provided by Docker:
-
-```sh
-docker compose up -d
-julia --project -e 'using Pkg; Pkg.test()'
-docker compose down
-```
+## Contributing
+
+The test suite manages its own temporary MySQL container via Harbor.jl. The only prerequisite is a working Docker daemon:
+
+```sh
+julia --project -e 'using Pkg; Pkg.test()'
+```

src/MySQL.jl

@@ -128,7 +128,7 @@ function setoptions!(mysql;
         ssl_crl::Union{AbstractString, Nothing}=nothing,
         ssl_crlpath::Union{AbstractString, Nothing}=nothing,
         passphrase::Union{AbstractString, Nothing}=nothing,
-        ssl_verify_server_cert::Union{Bool, Nothing}=nothing,
+        ssl_verify_server_cert::Union{Bool, Nothing}=false,
         ssl_enforce::Union{Bool, Nothing}=nothing,
         ssl_mode::Union{API.mysql_ssl_mode, Nothing}=nothing,
         default_auth::Union{AbstractString, Nothing}=nothing,
@@ -279,7 +279,7 @@ Connect to a MySQL database with provided `host`, `user`, and `passwd` positiona
   * `ssl_cipher::AbstractString`: Defines a list of permitted ciphers or cipher suites to use for TLS, like `"DHE-RSA-AES256-SHA"`
   * `ssl_crl::AbstractString`: Defines a path to a PEM file that should contain one or more revoked X509 certificates to use for TLS. This option requires that you use the absolute path, not a relative path.
   * `ssl_crlpath::AbstractString`: Defines a path to a directory that contains one or more PEM files that should each contain one revoked X509 certificate to use for TLS. This option requires that you use the absolute path, not a relative path. The directory specified by this option needs to be run through the openssl rehash command.
-  * `ssl_verify_server_cert::Bool`: Enables (or disables) server certificate verification.
+  * `ssl_verify_server_cert::Bool=false`: Enables (or disables) server certificate verification.
   * `ssl_enforce::Bool`: Whether to force TLS
   * `default_auth::AbstractString`: Default authentication client-side plugin to use.
   * `connection_handler::AbstractString`: Specify the name of a connection handler plugin.

src/api/capi.jl

@@ -431,13 +431,21 @@ Zero for success. Nonzero if an error occurred; this occurs for option values th
 """=#
 function getoption(mysql::MYSQL, option::mysql_option)
     if option in CUINTOPTS
-        return @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, option, Ref{Cuint}())
+        ref = Ref{Cuint}()
+        @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, Int(option), ref)
+        return ref[]
     elseif option in CULONGOPTS
-        return @checksuccess mysql mysql_get_option_Culong(mysql.ptr, option, Ref{Culong}())
+        ref = Ref{Culong}()
+        @checksuccess mysql mysql_get_option_Culong(mysql.ptr, Int(option), ref)
+        return ref[]
     elseif option in BOOLOPTS
-        return @checksuccess mysql mysql_get_option_Bool(mysql.ptr, option, Ref{Bool}())
+        ref = Ref{Bool}()
+        @checksuccess mysql mysql_get_option_Bool(mysql.ptr, Int(option), ref)
+        return ref[]
     else
-        return @checksuccess mysql mysql_get_option_String(mysql.ptr, option, Ref{String}())
+        ref = Ref{Ptr{UInt8}}(C_NULL)
+        @checksuccess mysql mysql_get_option_String(mysql.ptr, Int(option), ref)
+        return ref[] == C_NULL ? nothing : unsafe_string(ref[])
     end
 end
 

src/api/ccalls.jl

@@ -236,6 +236,13 @@ function mysql_get_option_Bool(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Bool
                 mysql, option, arg)
 end
 
+function mysql_get_option_String(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Ptr{UInt8}})
+    return @c(:mysql_get_option,
+                Cint,
+                (Ptr{Cvoid}, Cint, Ref{Ptr{UInt8}}),
+                mysql, option, arg)
+end
+
 function mysql_get_option_Cvoid(mysql::Ptr{Cvoid}, option::Integer, arg::Ptr{Cvoid})
     return @c(:mysql_get_option,
                 Cint,

src/execute.jl

@@ -203,8 +203,9 @@ Base.IteratorSize(::Type{<:TextCursors}) = Base.SizeUnknown()
 function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered}
     cursor.cursor.result.ptr == C_NULL && return nothing
     if !first
+        has_more_results = API.moreresults(cursor.cursor.conn.mysql)
         finalize(cursor.cursor.result)
-        if API.moreresults(cursor.cursor.conn.mysql)
+        if has_more_results
             @assert API.nextresult(cursor.cursor.conn.mysql) !== nothing
             cursor.cursor.result = buffered ? API.storeresult(cursor.cursor.conn.mysql) : API.useresult(cursor.cursor.conn.mysql)
             if buffered
@@ -222,4 +223,4 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered
 end
 
 DBInterface.executemultiple(conn::Connection, sql::AbstractString, params=(); kw...) =
-    TextCursors(DBInterface.execute(conn, sql, params; kw...))
+    TextCursors(DBInterface.execute(conn, sql, params; kw...))

src/load.jl

@@ -101,10 +101,14 @@ function load(itr, conn::Connection, name::AbstractString="mysql_"*Random.randst
     DBInterface.transaction(conn) do
         params = chop(repeat("?,", length(sch.names)))
         stmt = DBInterface.prepare(conn, "INSERT INTO $name ($(join(sch.names .|> string .|> quoteid,", "))) VALUES ($params)")
-        for (i, row) in enumerate(rows)
-            i > limit && break
-            debug && @info "inserting row $i; $(Tables.Row(row))"
-            DBInterface.execute(stmt, Tables.Row(row))
+        try
+            for (i, row) in enumerate(rows)
+                i > limit && break
+                debug && @info "inserting row $i; $(Tables.Row(row))"
+                DBInterface.execute(stmt, Tables.Row(row))
+            end
+        finally
+            DBInterface.close!(stmt)
         end
     end
 

test/runtests.jl

@@ -1,18 +1,142 @@
-using Test, MySQL, DBInterface, Tables, Dates, DecFP
+using Test, MySQL, DBInterface, Tables, Dates, DecFP, Harbor, Sockets
 
-conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+const MYSQL_IMAGE_REF = get(ENV, "MYSQL_IMAGE", "mysql:8")
+const MYSQL_TEST_USER = "root"
+const MYSQL_TEST_PASSWORD = ""
+
+struct MySQLTestConfig
+    host::String
+    port::Int
+    user::String
+    password::String
+end
+
+const TEST_CONFIG = Ref{Union{Nothing, MySQLTestConfig}}(nothing)
+const TEST_OPTION_FILE = Ref{Union{Nothing, String}}(nothing)
+
+function parse_image_ref(ref::String)
+    slash = findlast('/', ref)
+    colon = findlast(':', ref)
+    if colon !== nothing && (slash === nothing || colon > slash)
+        return String(ref[begin:prevind(ref, colon)]), String(ref[nextind(ref, colon):end])
+    end
+    return ref, "latest"
+end
+
+function docker_available()
+    Sys.which("docker") === nothing && return false
+    try
+        run(pipeline(`docker info`, stdout=devnull, stderr=devnull))
+        return true
+    catch
+        return false
+    end
+end
+
+function pick_port()
+    server = Sockets.listen(Sockets.IPv4(0), 0)
+    _, port = Sockets.getsockname(server)
+    port = Int(port)
+    close(server)
+    return port
+end
+
+test_config() = something(TEST_CONFIG[])
+test_host() = test_config().host
+test_port() = test_config().port
+test_user() = test_config().user
+test_password() = test_config().password
+test_option_file() = something(TEST_OPTION_FILE[])
+
+connect_mysql(; kw...) = DBInterface.connect(MySQL.Connection, test_host(), test_user(), test_password(); port=test_port(), kw...)
+
+function wait_for_connection(cfg::MySQLTestConfig; timeout::Float64=90.0)
+    start_time = time()
+    last_err = nothing
+    while time() - start_time < timeout
+        try
+            return DBInterface.connect(MySQL.Connection, cfg.host, cfg.user, cfg.password; port=cfg.port, connect_timeout=2)
+        catch err
+            last_err = err
+            sleep(0.5)
+        end
+    end
+    last_err === nothing && error("MySQL did not become ready")
+    error("MySQL did not become ready: $(sprint(showerror, last_err))")
+end
+
+function write_option_file(dir::AbstractString, cfg::MySQLTestConfig)
+    path = joinpath(dir, "my.ini")
+    open(path, "w") do io
+        print(io, """
+[client]
+host=$(cfg.host)
+user=$(cfg.user)
+port=$(cfg.port)
+connect_timeout=30
+report-data-truncation=true
+password = $(repr(cfg.password))
+""")
+    end
+    return path
+end
+
+function with_mysql(f::Function)
+    image, tag = parse_image_ref(MYSQL_IMAGE_REF)
+    host_port = pick_port()
+    env = Dict("MYSQL_ALLOW_EMPTY_PASSWORD" => "yes")
+    return Harbor.with_container(
+        image;
+        tag=tag,
+        ports=Dict(3306 => host_port),
+        environment=env,
+        wait_strategy=(pattern="ready for connections",),
+        wait_timeout=120.0,
+    ) do _
+        cfg = MySQLTestConfig("127.0.0.1", host_port, MYSQL_TEST_USER, MYSQL_TEST_PASSWORD)
+        conn = wait_for_connection(cfg)
+        DBInterface.close!(conn)
+        return f(cfg)
+    end
+end
+
+@testset "MySQL" begin
+
+let mysql = MySQL.API.init()
+    MySQL.setoptions!(mysql)
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == false
+    MySQL.setoptions!(mysql; ssl_verify_server_cert=true)
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == true
+    MySQL.setoptions!(mysql; connect_timeout=7)
+    @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_CONNECT_TIMEOUT)) == 7
+    MySQL.setoptions!(mysql; max_allowed_packet=1024)
+    @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_MAX_ALLOWED_PACKET)) == 1024
+    MySQL.setoptions!(mysql; bind="127.0.0.1")
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_BIND) == "127.0.0.1"
+end
+
+if !docker_available()
+    @info "Docker not available; skipping MySQL integration tests."
+    @test true
+else
+    with_mysql() do cfg
+        TEST_CONFIG[] = cfg
+        mktempdir() do dir
+            TEST_OPTION_FILE[] = write_option_file(dir, cfg)
+
+conn = connect_mysql()
 DBInterface.close!(conn)
 
 # https://github.com/JuliaDatabases/MySQL.jl/issues/170
-conn = DBInterface.connect(MySQL.Connection, "mysql://127.0.0.1", "root"; port=3306)
+conn = DBInterface.connect(MySQL.Connection, string("mysql://", test_host()), test_user(); port=test_port())
 DBInterface.close!(conn)
 
 # AbstractString as a connection parameter or an option
-conn = DBInterface.connect(MySQL.Connection, SubString("127.0.0.1"), SubString("root"), SubString(""); port=3306, charset_name=SubString("utf8mb4"))
+conn = DBInterface.connect(MySQL.Connection, SubString(test_host()), SubString(test_user()), SubString(test_password()); port=test_port(), charset_name=SubString("utf8mb4"))
 DBInterface.close!(conn)
 
 # load host/user + options from file
-conn = DBInterface.connect(MySQL.Connection, "", ""; option_file=joinpath(dirname(pathof(MySQL)), "../test/", "my.ini"))
+conn = DBInterface.connect(MySQL.Connection, "", ""; port=0, option_file=test_option_file())
 @test isopen(conn)
 
 DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
@@ -310,7 +434,8 @@ res = DBInterface.execute(stmt) |> columntable
 res = DBInterface.execute(stmt)
 res = DBInterface.execute(stmt)
 
-results = DBInterface.executemultiple(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
+multi_conn = connect_mysql(db="mysqltest")
+results = DBInterface.executemultiple(multi_conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
 state = iterate(results)
 @test state !== nothing
 res, st = state
@@ -325,6 +450,7 @@ res, st = state
 @test length(res) == 4
 ret = columntable(res)
 @test length(ret[1]) == 4
+DBInterface.close!(multi_conn)
 
 # multiple-queries not supported by mysql w/ prepared statements
 @test_throws MySQL.API.StmtError DBInterface.prepare(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
@@ -372,15 +498,15 @@ ret = columntable(res)
 @test_throws ArgumentError MySQL.load(ct, conn, "test194")
 
 @testset "transactions" begin
-    conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+    conn = connect_mysql()
     try
         DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
         DBInterface.execute(conn, "CREATE DATABASE mysqltest")
         DBInterface.execute(conn, "use mysqltest")
         DBInterface.execute(conn, "DROP TABLE IF EXISTS TransactionTest")
         DBInterface.execute(conn, "CREATE TABLE TransactionTest (a int)")
 
-        conn2 = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+        conn2 = connect_mysql()
         DBInterface.execute(conn2, "use mysqltest")
 
         try
@@ -414,4 +540,8 @@ ret = columntable(res)
     finally
         DBInterface.close!(conn)
     end
+end
+        end
+    end
+end
 end