From 20f1cf71d865105bf60f4c11483a912395be18da Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 28 May 2024 20:04:27 +0200 Subject: [PATCH] feat(socket): work without luasocket, using luasystem instead (#171) All we need for the core scheduling tasks is: - a `sleep` function for a non-busy-loop wait - a `gettime` function to get actual time at usecond So without using sockets, luasystem should be good enough. Removing LuaSocket from the rockspec is a breaking change, hence it is still installed by default. To be reviewed in a next major. --- .github/workflows/unix_build.yml | 1 + Makefile | 1 + docs/index.html | 12 +++- docs/reference.html | 6 ++ src/copas.lua | 120 +++++++++++++++++++------------ src/copas/lock.lua | 2 +- src/copas/queue.lua | 2 +- tests/close.lua | 6 +- tests/largetransfer.lua | 14 ++-- tests/lock.lua | 2 +- tests/no_luasocket.lua | 44 ++++++++++++ tests/queue.lua | 2 +- tests/removethread.lua | 2 +- tests/semaphore.lua | 2 +- tests/starve.lua | 8 +-- tests/timer.lua | 2 +- 16 files changed, 155 insertions(+), 71 deletions(-) create mode 100644 tests/no_luasocket.lua diff --git a/.github/workflows/unix_build.yml b/.github/workflows/unix_build.yml index 84af447..a99cfca 100644 --- a/.github/workflows/unix_build.yml +++ b/.github/workflows/unix_build.yml @@ -37,6 +37,7 @@ jobs: run: | luarocks install luacov-coveralls luarocks install luasec + luarocks install luasystem - name: generate test certificates run: | diff --git a/Makefile b/Makefile index 7c0b584..de8dbe9 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ test: certs $(LUA) $(DELIM) $(PKGPATH) tests/largetransfer.lua $(LUA) $(DELIM) $(PKGPATH) tests/lock.lua $(LUA) $(DELIM) $(PKGPATH) tests/loop_starter.lua + $(LUA) $(DELIM) $(PKGPATH) tests/no_luasocket.lua $(LUA) $(DELIM) $(PKGPATH) tests/pause.lua $(LUA) $(DELIM) $(PKGPATH) tests/queue.lua $(LUA) $(DELIM) $(PKGPATH) tests/removeserver.lua diff --git a/docs/index.html b/docs/index.html index 4c08240..95d6090 100644 --- a/docs/index.html +++ b/docs/index.html @@ -93,13 +93,19 @@

Download

Dependencies

-

Copas depends on -LuaSocket, Coxpcall (only when using Lua 5.1), and (optionally) LuaSec. -

+

Copas depends on LuaSocket (or LuaSystem), Coxpcall +(only when using Lua 5.1), and (optionally) LuaSec.

History

+
Copas 4.8.0 unreleased
+
    +
  • Change: Copas no longer requires LuaSocket, if no sockets are needed, LuaSystem will be enough as a fallback.
    • +
  • Feat: added copas.gettime(), which transparently maps to either LuaSockets or + LuaSystems implementation, ensuring independence of the availability of either one of those.
    • +
+
Copas 4.7.1 [9/Mar/2024]
  • Fix: copas.removethread would not remove a sleeping thread immediately (it would not execute, but diff --git a/docs/reference.html b/docs/reference.html index 843416e..6795a20 100644 --- a/docs/reference.html +++ b/docs/reference.html @@ -210,6 +210,12 @@

    Copas dispatcher main functions

    currently running coroutine.

+
number = copas.gettime()
+
+

Returns the (fractional) number of seconds since the epoch. This directly + maps to either the LuaSocket or LuaSystem implementation of gettime().

+
+
string = copas.gettraceback([msg], [co], [skt])

Creates a traceback (string). Can be used from custom errorhandlers to create diff --git a/src/copas.lua b/src/copas.lua index 0c79237..19a195c 100644 --- a/src/copas.lua +++ b/src/copas.lua @@ -20,14 +20,27 @@ if package.loaded["copas.http"] and (_VERSION=="Lua 5.1") then -- obsolete: error("you must require copas before require'ing copas.http") end +-- load either LuaSocket, or LuaSystem +local socket, system do + if pcall(require, "socket") then + -- found LuaSocket + socket = require "socket" + else + -- fallback to LuaSystem + if pcall(require, "system") then + system = require "system" + else + error("Neither LuaSocket nor LuaSystem found, Copas requires at least one of them") + end + end +end -local socket = require "socket" local binaryheap = require "binaryheap" -local gettime = socket.gettime +local gettime = (socket or system).gettime local ssl -- only loaded upon demand local WATCH_DOG_TIMEOUT = 120 -local UDP_DATAGRAM_MAX = socket._DATAGRAMSIZE or 8192 +local UDP_DATAGRAM_MAX = (socket or {})._DATAGRAMSIZE or 8192 local TIMEOUT_PRECISION = 0.1 -- 100ms local fnil = function() end @@ -52,7 +65,7 @@ if _VERSION=="Lua 5.1" and not jit then -- obsolete: only for Lua 5.1 compat end -do +if socket then -- Redefines LuaSocket functions with coroutine safe versions (pure Lua) -- (this allows the use of socket.http from within copas) local err_mt = { @@ -126,6 +139,8 @@ copas.autoclose = true -- indicator for the loop running copas.running = false +-- gettime method from either LuaSocket or LuaSystem: time in (fractional) seconds, since epoch. +copas.gettime = gettime ------------------------------------------------------------------------------- -- Object names, to track names of thread/coroutines and sockets @@ -1355,6 +1370,7 @@ do local timeout_register = setmetatable({}, { __mode = "k" }) local time_out_thread local timerwheel = require("timerwheel").new({ + now = gettime, precision = TIMEOUT_PRECISION, ringsize = math.floor(60*60*24/TIMEOUT_PRECISION), -- ring size 1 day err_handler = function(err) @@ -1416,6 +1432,8 @@ end -- a task to check ready to read events local _readable_task = {} do + _readable_task._events = {} + local function tick(skt) local handler = _servers[skt] if handler then @@ -1439,6 +1457,8 @@ end -- a task to check ready to write events local _writable_task = {} do + _writable_task._events = {} + local function tick(skt) _writing:remove(skt) _doTick(_writing:pop(skt), skt) @@ -1502,59 +1522,65 @@ local _select_plain do local last_cleansing = 0 local duration = function(t2, t1) return t2-t1 end - _select_plain = function(timeout) - local err - local now = gettime() - - -- remove any closed sockets to prevent select from hanging on them - if _closed[1] then - for i, skt in ipairs(_closed) do - _closed[i] = { _reading:remove(skt), _writing:remove(skt) } + if not socket then + -- socket module unavailable, switch to luasystem sleep + _select_plain = system.sleep + else + -- use socket.select to handle socket-io + _select_plain = function(timeout) + local err + local now = gettime() + + -- remove any closed sockets to prevent select from hanging on them + if _closed[1] then + for i, skt in ipairs(_closed) do + _closed[i] = { _reading:remove(skt), _writing:remove(skt) } + end end - end - _readable_task._events, _writable_task._events, err = socket.select(_reading, _writing, timeout) - local r_events, w_events = _readable_task._events, _writable_task._events + _readable_task._events, _writable_task._events, err = socket.select(_reading, _writing, timeout) + local r_events, w_events = _readable_task._events, _writable_task._events - -- inject closed sockets in readable/writeable task so they can error out properly - if _closed[1] then - for i, skts in ipairs(_closed) do - _closed[i] = nil - r_events[#r_events+1] = skts[1] - w_events[#w_events+1] = skts[2] + -- inject closed sockets in readable/writeable task so they can error out properly + if _closed[1] then + for i, skts in ipairs(_closed) do + _closed[i] = nil + r_events[#r_events+1] = skts[1] + w_events[#w_events+1] = skts[2] + end end - end - if duration(now, last_cleansing) > WATCH_DOG_TIMEOUT then - last_cleansing = now - - -- Check all sockets selected for reading, and check how long they have been waiting - -- for data already, without select returning them as readable - for skt,time in pairs(_reading_log) do - if not r_events[skt] and duration(now, time) > WATCH_DOG_TIMEOUT then - -- This one timedout while waiting to become readable, so move - -- it in the readable list and try and read anyway, despite not - -- having been returned by select - _reading_log[skt] = nil - r_events[#r_events + 1] = skt - r_events[skt] = #r_events + if duration(now, last_cleansing) > WATCH_DOG_TIMEOUT then + last_cleansing = now + + -- Check all sockets selected for reading, and check how long they have been waiting + -- for data already, without select returning them as readable + for skt,time in pairs(_reading_log) do + if not r_events[skt] and duration(now, time) > WATCH_DOG_TIMEOUT then + -- This one timedout while waiting to become readable, so move + -- it in the readable list and try and read anyway, despite not + -- having been returned by select + _reading_log[skt] = nil + r_events[#r_events + 1] = skt + r_events[skt] = #r_events + end end - end - -- Do the same for writing - for skt,time in pairs(_writing_log) do - if not w_events[skt] and duration(now, time) > WATCH_DOG_TIMEOUT then - _writing_log[skt] = nil - w_events[#w_events + 1] = skt - w_events[skt] = #w_events + -- Do the same for writing + for skt,time in pairs(_writing_log) do + if not w_events[skt] and duration(now, time) > WATCH_DOG_TIMEOUT then + _writing_log[skt] = nil + w_events[#w_events + 1] = skt + w_events[skt] = #w_events + end end end - end - if err == "timeout" and #r_events + #w_events > 0 then - return nil - else - return err + if err == "timeout" and #r_events + #w_events > 0 then + return nil + else + return err + end end end end diff --git a/src/copas/lock.lua b/src/copas/lock.lua index 7b33c0d..68dfaea 100644 --- a/src/copas/lock.lua +++ b/src/copas/lock.lua @@ -1,5 +1,5 @@ local copas = require("copas") -local gettime = require("socket").gettime +local gettime = copas.gettime local DEFAULT_TIMEOUT = 10 diff --git a/src/copas/queue.lua b/src/copas/queue.lua index ff79892..4e3126c 100644 --- a/src/copas/queue.lua +++ b/src/copas/queue.lua @@ -1,5 +1,5 @@ local copas = require "copas" -local gettime = require("socket").gettime +local gettime = copas.gettime local Sema = copas.semaphore local Lock = copas.lock diff --git a/tests/close.lua b/tests/close.lua index cbc31b3..b00a9f1 100644 --- a/tests/close.lua +++ b/tests/close.lua @@ -36,7 +36,7 @@ copas.loop(function() -- wait in the read/write queues copas.pause(2) -- now we're closing the connecting_socket - close_time = socket.gettime() + close_time = copas.gettime() print("closing client socket now, client receive and send operation should immediately error out now") client_socket:close() @@ -55,7 +55,7 @@ copas.loop(function() copas.addthread(function() local data, err = client_socket:receive(1) print("receive result: ", tostring(data), tostring(err)) - receive_end_time = socket.gettime() + receive_end_time = copas.gettime() print("receive took: ", receive_end_time - close_time) check_exit() end) @@ -66,7 +66,7 @@ copas.loop(function() ok, err = client_socket:send(("hello world"):rep(100)) end print("send result: ", tostring(ok), tostring(err)) - send_end_time = socket.gettime() + send_end_time = copas.gettime() print("send took: ", send_end_time - close_time) check_exit() end) diff --git a/tests/largetransfer.lua b/tests/largetransfer.lua index 6f9d556..46363d6 100644 --- a/tests/largetransfer.lua +++ b/tests/largetransfer.lua @@ -12,7 +12,7 @@ local socket = require 'socket' -- copas.debug.start() local body = ("A"):rep(1024*1024*50) -- 50 mb string -local start = socket.gettime() +local start = copas.gettime() local done = 0 local sparams, cparams @@ -25,7 +25,7 @@ local function runtest() local res, err, part = skt:receive('*a') res = res or part if res ~= body then print("Received doesn't match send") end - print("Server reading port 49500... Done!", socket.gettime()-start, err, #res) + print("Server reading port 49500... Done!", copas.gettime()-start, err, #res) copas.removeserver(s1) done = done + 1 end, sparams)) @@ -38,7 +38,7 @@ local function runtest() local res, err, part = skt:receive('*a') res = res or part if res ~= body then print("Received doesn't match send") end - print("Server reading port 49501... Done!", socket.gettime()-start, err, #res) + print("Server reading port 49501... Done!", copas.gettime()-start, err, #res) copas.removeserver(s2) done = done + 1 end, sparams)) @@ -52,7 +52,7 @@ local function runtest() repeat last_byte_sent, err = skt:send(body, last_byte_sent or 1, -1) until last_byte_sent == nil or last_byte_sent == #body - print("Client writing port 49500... Done!", socket.gettime()-start, err, #body) + print("Client writing port 49500... Done!", copas.gettime()-start, err, #body) -- we're not closing the socket, so the Copas GC-when-idle can kick-in to clean up skt = nil -- luacheck: ignore done = done + 1 @@ -67,7 +67,7 @@ local function runtest() repeat last_byte_sent, err = skt:send(body, last_byte_sent or 1, -1) until last_byte_sent == nil or last_byte_sent == #body - print("Client writing port 49501... Done!", socket.gettime()-start, err, #body) + print("Client writing port 49501... Done!", copas.gettime()-start, err, #body) -- we're not closing the socket, so the Copas GC-when-idle can kick-in to clean up skt = nil -- luacheck: ignore done = done + 1 @@ -77,7 +77,7 @@ local function runtest() local i = 1 while done ~= 4 do copas.pause(1) - print(i, "seconds:", socket.gettime()-start) + print(i, "seconds:", copas.gettime()-start) i = i + 1 if i > 60 then print"timeout" @@ -114,5 +114,5 @@ cparams = { options = {"all", "no_sslv2", "no_sslv3", "no_tlsv1"}, } done = 0 -start = socket.gettime() +start = copas.gettime() runtest() diff --git a/tests/lock.lua b/tests/lock.lua index 0754053..850d8c8 100644 --- a/tests/lock.lua +++ b/tests/lock.lua @@ -5,7 +5,7 @@ package.path = string.format("../src/?.lua;%s", package.path) local copas = require "copas" local Lock = copas.lock -local gettime = require("socket").gettime +local gettime = copas.gettime local test_complete = false copas.loop(function() diff --git a/tests/no_luasocket.lua b/tests/no_luasocket.lua new file mode 100644 index 0000000..167ccc8 --- /dev/null +++ b/tests/no_luasocket.lua @@ -0,0 +1,44 @@ +-- make sure we are pointing to the local copas first +package.path = string.format("../src/?.lua;%s", package.path) + +print([[ +Testing to run Copas without LuaSocket, just LuaSystem +============================================================================= +]]) + +-- patch require to no longer load luasocket +local _require = require +_G.require = function(name) + if name == "socket" then + error("luasocket is not allowed in this test") + end + return _require(name) +end + + +local copas = require "copas" +local timer = copas.timer + +local successes = 0 + +local t1 -- luacheck: ignore + +copas.loop(function() + + t1 = timer.new({ + delay = 0.1, + recurring = true, + callback = function(timer_obj, params) + successes = successes + 1 -- 6 to come + if successes == 6 then + timer_obj:cancel() + end + end, + }) + -- succes count = 6 + +end) + + +assert(successes == 6, "number of successes didn't match! got: "..successes) +print("test success!") diff --git a/tests/queue.lua b/tests/queue.lua index ef0b9e9..513ff99 100644 --- a/tests/queue.lua +++ b/tests/queue.lua @@ -1,9 +1,9 @@ -- make sure we are pointing to the local copas first package.path = string.format("../src/?.lua;%s", package.path) -local now = require("socket").gettime local copas = require "copas" +local now = copas.gettime local Queue = copas.queue diff --git a/tests/removethread.lua b/tests/removethread.lua index 7c4dd7e..88dc711 100644 --- a/tests/removethread.lua +++ b/tests/removethread.lua @@ -4,7 +4,7 @@ package.path = string.format("../src/?.lua;%s", package.path) local copas = require("copas") -local now = require("socket").gettime +local now = copas.gettime -- Test 1: basic test diff --git a/tests/semaphore.lua b/tests/semaphore.lua index e922014..c5d3865 100644 --- a/tests/semaphore.lua +++ b/tests/semaphore.lua @@ -1,9 +1,9 @@ -- make sure we are pointing to the local copas first package.path = string.format("../src/?.lua;%s", package.path) -local now = require("socket").gettime local copas = require "copas" +local now = copas.gettime local semaphore = copas.semaphore diff --git a/tests/starve.lua b/tests/starve.lua index 6d07d3b..4b70667 100644 --- a/tests/starve.lua +++ b/tests/starve.lua @@ -20,8 +20,8 @@ local function runtest() copas.setsocketname("Server 49500", skt) copas.setthreadname("Server 49500") print "Server 49500 accepted incoming connection" - local end_time = socket.gettime() + 30 -- we run for 30 seconds - while end_time > socket.gettime() do + local end_time = copas.gettime() + 30 -- we run for 30 seconds + while end_time > copas.gettime() do local res, err, _ = skt:receive(1) -- single byte from 50mb chunks if res == nil and err ~= "timeout" then print("Server 49500 returned: " .. err) @@ -61,11 +61,11 @@ local function runtest() copas.addnamedthread("test timeout thread", function() local i = 0 - local start = socket.gettime() + local start = copas.gettime() while done ~= 2 do copas.pause(1) -- delta sleep, so it slowly diverges if starved i = i + 1 - local time_passed = socket.gettime()-start + local time_passed = copas.gettime()-start print("slept "..i.." seconds, time passed: ".. time_passed.." seconds") if math.abs(i - time_passed) > 2 then print("timer diverged by more than 2 seconds: failed!") diff --git a/tests/timer.lua b/tests/timer.lua index 833db4a..e4d4420 100644 --- a/tests/timer.lua +++ b/tests/timer.lua @@ -4,7 +4,7 @@ package.path = string.format("../src/?.lua;%s", package.path) local copas = require "copas" -local gettime = require("socket").gettime +local gettime = copas.gettime local timer = copas.timer local successes = 0