diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c76c290..86e2a30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: version: - - "1.6" + - "1.10" - 1 # automatically expands to the latest stable 1.x release of Julia - nightly os: @@ -21,7 +21,7 @@ jobs: - x64 steps: - uses: actions/checkout@v5 - - run: docker compose up -d + - run: docker info - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} @@ -50,4 +50,4 @@ jobs: - run: julia --project=docs docs/make.jl env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} \ No newline at end of file + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/Project.toml b/Project.toml index 36af441..e8a2aa5 100644 --- a/Project.toml +++ b/Project.toml @@ -17,14 +17,17 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] DBInterface = "2.5" DecFP = "0.4.9, 0.4.10, 1" +Harbor = "1.0.3" MariaDB_Connector_C_jll = "3.1.12" OpenSSL_jll = "3" Parsers = "0.3, 1, 2" Tables = "1" -julia = "1.6" +julia = "1.10" [extras] +Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["Harbor", "Sockets", "Test"] diff --git a/README.md b/README.md index 1d93b7e..269b414 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,10 @@ Package for interfacing with MySQL databases from Julia via the MariaDB C connec [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://mysql.juliadatabases.org/stable) [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://mysql.juliadatabases.org/dev) -## Contributing - -The tests require a MySQL DB to be running, which is provided by Docker: - -```sh -docker compose up -d -julia --project -e 'using Pkg; Pkg.test()' -docker compose down -``` +## Contributing + +The test suite manages its own temporary MySQL container via Harbor.jl. The only prerequisite is a working Docker daemon: + +```sh +julia --project -e 'using Pkg; Pkg.test()' +``` diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index eea79f9..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.9" -name: "mysqljl-test" -services: - db: - image: mysql:8 - ports: - - 3306:3306 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true - healthcheck: - test: - [ - "CMD", - "mysql", - "-u", - "root", - "-p''", - "--silent", - "--execute", - "SELECT 1;", - ] - interval: 30s - timeout: 10s - retries: 5 - networks: - - app -networks: - app: - driver: bridge diff --git a/src/MySQL.jl b/src/MySQL.jl index 13fe90e..05bab5a 100644 --- a/src/MySQL.jl +++ b/src/MySQL.jl @@ -128,7 +128,7 @@ function setoptions!(mysql; ssl_crl::Union{AbstractString, Nothing}=nothing, ssl_crlpath::Union{AbstractString, Nothing}=nothing, passphrase::Union{AbstractString, Nothing}=nothing, - ssl_verify_server_cert::Union{Bool, Nothing}=nothing, + ssl_verify_server_cert::Union{Bool, Nothing}=false, ssl_enforce::Union{Bool, Nothing}=nothing, ssl_mode::Union{API.mysql_ssl_mode, Nothing}=nothing, default_auth::Union{AbstractString, Nothing}=nothing, @@ -279,7 +279,7 @@ Connect to a MySQL database with provided `host`, `user`, and `passwd` positiona * `ssl_cipher::AbstractString`: Defines a list of permitted ciphers or cipher suites to use for TLS, like `"DHE-RSA-AES256-SHA"` * `ssl_crl::AbstractString`: Defines a path to a PEM file that should contain one or more revoked X509 certificates to use for TLS. This option requires that you use the absolute path, not a relative path. * `ssl_crlpath::AbstractString`: Defines a path to a directory that contains one or more PEM files that should each contain one revoked X509 certificate to use for TLS. This option requires that you use the absolute path, not a relative path. The directory specified by this option needs to be run through the openssl rehash command. - * `ssl_verify_server_cert::Bool`: Enables (or disables) server certificate verification. + * `ssl_verify_server_cert::Bool=false`: Enables (or disables) server certificate verification. * `ssl_enforce::Bool`: Whether to force TLS * `default_auth::AbstractString`: Default authentication client-side plugin to use. * `connection_handler::AbstractString`: Specify the name of a connection handler plugin. diff --git a/src/api/capi.jl b/src/api/capi.jl index d85382f..7acbc50 100644 --- a/src/api/capi.jl +++ b/src/api/capi.jl @@ -431,13 +431,21 @@ Zero for success. Nonzero if an error occurred; this occurs for option values th """=# function getoption(mysql::MYSQL, option::mysql_option) if option in CUINTOPTS - return @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, option, Ref{Cuint}()) + ref = Ref{Cuint}() + @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, Int(option), ref) + return ref[] elseif option in CULONGOPTS - return @checksuccess mysql mysql_get_option_Culong(mysql.ptr, option, Ref{Culong}()) + ref = Ref{Culong}() + @checksuccess mysql mysql_get_option_Culong(mysql.ptr, Int(option), ref) + return ref[] elseif option in BOOLOPTS - return @checksuccess mysql mysql_get_option_Bool(mysql.ptr, option, Ref{Bool}()) + ref = Ref{Bool}() + @checksuccess mysql mysql_get_option_Bool(mysql.ptr, Int(option), ref) + return ref[] else - return @checksuccess mysql mysql_get_option_String(mysql.ptr, option, Ref{String}()) + ref = Ref{Ptr{UInt8}}(C_NULL) + @checksuccess mysql mysql_get_option_String(mysql.ptr, Int(option), ref) + return ref[] == C_NULL ? nothing : unsafe_string(ref[]) end end diff --git a/src/api/ccalls.jl b/src/api/ccalls.jl index aad0729..95fc51c 100644 --- a/src/api/ccalls.jl +++ b/src/api/ccalls.jl @@ -236,6 +236,13 @@ function mysql_get_option_Bool(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Bool mysql, option, arg) end +function mysql_get_option_String(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Ptr{UInt8}}) + return @c(:mysql_get_option, + Cint, + (Ptr{Cvoid}, Cint, Ref{Ptr{UInt8}}), + mysql, option, arg) +end + function mysql_get_option_Cvoid(mysql::Ptr{Cvoid}, option::Integer, arg::Ptr{Cvoid}) return @c(:mysql_get_option, Cint, diff --git a/src/execute.jl b/src/execute.jl index b1d2816..5dcabd3 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -203,8 +203,9 @@ Base.IteratorSize(::Type{<:TextCursors}) = Base.SizeUnknown() function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered} cursor.cursor.result.ptr == C_NULL && return nothing if !first + has_more_results = API.moreresults(cursor.cursor.conn.mysql) finalize(cursor.cursor.result) - if API.moreresults(cursor.cursor.conn.mysql) + if has_more_results @assert API.nextresult(cursor.cursor.conn.mysql) !== nothing cursor.cursor.result = buffered ? API.storeresult(cursor.cursor.conn.mysql) : API.useresult(cursor.cursor.conn.mysql) if buffered @@ -222,4 +223,4 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered end DBInterface.executemultiple(conn::Connection, sql::AbstractString, params=(); kw...) = - TextCursors(DBInterface.execute(conn, sql, params; kw...)) \ No newline at end of file + TextCursors(DBInterface.execute(conn, sql, params; kw...)) diff --git a/src/load.jl b/src/load.jl index d6c3895..eac9beb 100644 --- a/src/load.jl +++ b/src/load.jl @@ -101,10 +101,14 @@ function load(itr, conn::Connection, name::AbstractString="mysql_"*Random.randst DBInterface.transaction(conn) do params = chop(repeat("?,", length(sch.names))) stmt = DBInterface.prepare(conn, "INSERT INTO $name ($(join(sch.names .|> string .|> quoteid,", "))) VALUES ($params)") - for (i, row) in enumerate(rows) - i > limit && break - debug && @info "inserting row $i; $(Tables.Row(row))" - DBInterface.execute(stmt, Tables.Row(row)) + try + for (i, row) in enumerate(rows) + i > limit && break + debug && @info "inserting row $i; $(Tables.Row(row))" + DBInterface.execute(stmt, Tables.Row(row)) + end + finally + DBInterface.close!(stmt) end end diff --git a/test/my.ini b/test/my.ini deleted file mode 100644 index 07e5400..0000000 --- a/test/my.ini +++ /dev/null @@ -1,7 +0,0 @@ -[client] -host=127.0.0.1 -user=root -port=3306 -connect_timeout=30 -report-data-truncation=true -password = "" \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 436471b..061983f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,18 +1,142 @@ -using Test, MySQL, DBInterface, Tables, Dates, DecFP +using Test, MySQL, DBInterface, Tables, Dates, DecFP, Harbor, Sockets -conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306) +const MYSQL_IMAGE_REF = get(ENV, "MYSQL_IMAGE", "mysql:8") +const MYSQL_TEST_USER = "root" +const MYSQL_TEST_PASSWORD = "" + +struct MySQLTestConfig + host::String + port::Int + user::String + password::String +end + +const TEST_CONFIG = Ref{Union{Nothing, MySQLTestConfig}}(nothing) +const TEST_OPTION_FILE = Ref{Union{Nothing, String}}(nothing) + +function parse_image_ref(ref::String) + slash = findlast('/', ref) + colon = findlast(':', ref) + if colon !== nothing && (slash === nothing || colon > slash) + return String(ref[begin:prevind(ref, colon)]), String(ref[nextind(ref, colon):end]) + end + return ref, "latest" +end + +function docker_available() + Sys.which("docker") === nothing && return false + try + run(pipeline(`docker info`, stdout=devnull, stderr=devnull)) + return true + catch + return false + end +end + +function pick_port() + server = Sockets.listen(Sockets.IPv4(0), 0) + _, port = Sockets.getsockname(server) + port = Int(port) + close(server) + return port +end + +test_config() = something(TEST_CONFIG[]) +test_host() = test_config().host +test_port() = test_config().port +test_user() = test_config().user +test_password() = test_config().password +test_option_file() = something(TEST_OPTION_FILE[]) + +connect_mysql(; kw...) = DBInterface.connect(MySQL.Connection, test_host(), test_user(), test_password(); port=test_port(), kw...) + +function wait_for_connection(cfg::MySQLTestConfig; timeout::Float64=90.0) + start_time = time() + last_err = nothing + while time() - start_time < timeout + try + return DBInterface.connect(MySQL.Connection, cfg.host, cfg.user, cfg.password; port=cfg.port, connect_timeout=2) + catch err + last_err = err + sleep(0.5) + end + end + last_err === nothing && error("MySQL did not become ready") + error("MySQL did not become ready: $(sprint(showerror, last_err))") +end + +function write_option_file(dir::AbstractString, cfg::MySQLTestConfig) + path = joinpath(dir, "my.ini") + open(path, "w") do io + print(io, """ +[client] +host=$(cfg.host) +user=$(cfg.user) +port=$(cfg.port) +connect_timeout=30 +report-data-truncation=true +password = $(repr(cfg.password)) +""") + end + return path +end + +function with_mysql(f::Function) + image, tag = parse_image_ref(MYSQL_IMAGE_REF) + host_port = pick_port() + env = Dict("MYSQL_ALLOW_EMPTY_PASSWORD" => "yes") + return Harbor.with_container( + image; + tag=tag, + ports=Dict(3306 => host_port), + environment=env, + wait_strategy=(pattern="ready for connections",), + wait_timeout=120.0, + ) do _ + cfg = MySQLTestConfig("127.0.0.1", host_port, MYSQL_TEST_USER, MYSQL_TEST_PASSWORD) + conn = wait_for_connection(cfg) + DBInterface.close!(conn) + return f(cfg) + end +end + +@testset "MySQL" begin + +let mysql = MySQL.API.init() + MySQL.setoptions!(mysql) + @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == false + MySQL.setoptions!(mysql; ssl_verify_server_cert=true) + @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == true + MySQL.setoptions!(mysql; connect_timeout=7) + @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_CONNECT_TIMEOUT)) == 7 + MySQL.setoptions!(mysql; max_allowed_packet=1024) + @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_MAX_ALLOWED_PACKET)) == 1024 + MySQL.setoptions!(mysql; bind="127.0.0.1") + @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_BIND) == "127.0.0.1" +end + +if !docker_available() + @info "Docker not available; skipping MySQL integration tests." + @test true +else + with_mysql() do cfg + TEST_CONFIG[] = cfg + mktempdir() do dir + TEST_OPTION_FILE[] = write_option_file(dir, cfg) + +conn = connect_mysql() DBInterface.close!(conn) # https://github.com/JuliaDatabases/MySQL.jl/issues/170 -conn = DBInterface.connect(MySQL.Connection, "mysql://127.0.0.1", "root"; port=3306) +conn = DBInterface.connect(MySQL.Connection, string("mysql://", test_host()), test_user(); port=test_port()) DBInterface.close!(conn) # AbstractString as a connection parameter or an option -conn = DBInterface.connect(MySQL.Connection, SubString("127.0.0.1"), SubString("root"), SubString(""); port=3306, charset_name=SubString("utf8mb4")) +conn = DBInterface.connect(MySQL.Connection, SubString(test_host()), SubString(test_user()), SubString(test_password()); port=test_port(), charset_name=SubString("utf8mb4")) DBInterface.close!(conn) # load host/user + options from file -conn = DBInterface.connect(MySQL.Connection, "", ""; option_file=joinpath(dirname(pathof(MySQL)), "../test/", "my.ini")) +conn = DBInterface.connect(MySQL.Connection, "", ""; port=0, option_file=test_option_file()) @test isopen(conn) DBInterface.execute(conn, "DROP DATABASE if exists mysqltest") @@ -310,7 +434,8 @@ res = DBInterface.execute(stmt) |> columntable res = DBInterface.execute(stmt) res = DBInterface.execute(stmt) -results = DBInterface.executemultiple(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL") +multi_conn = connect_mysql(db="mysqltest") +results = DBInterface.executemultiple(multi_conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL") state = iterate(results) @test state !== nothing res, st = state @@ -325,6 +450,7 @@ res, st = state @test length(res) == 4 ret = columntable(res) @test length(ret[1]) == 4 +DBInterface.close!(multi_conn) # multiple-queries not supported by mysql w/ prepared statements @test_throws MySQL.API.StmtError DBInterface.prepare(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL") @@ -372,7 +498,7 @@ ret = columntable(res) @test_throws ArgumentError MySQL.load(ct, conn, "test194") @testset "transactions" begin - conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306) + conn = connect_mysql() try DBInterface.execute(conn, "DROP DATABASE if exists mysqltest") DBInterface.execute(conn, "CREATE DATABASE mysqltest") @@ -380,7 +506,7 @@ ret = columntable(res) DBInterface.execute(conn, "DROP TABLE IF EXISTS TransactionTest") DBInterface.execute(conn, "CREATE TABLE TransactionTest (a int)") - conn2 = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306) + conn2 = connect_mysql() DBInterface.execute(conn2, "use mysqltest") try @@ -414,4 +540,8 @@ ret = columntable(res) finally DBInterface.close!(conn) end +end + end + end +end end