I am attempting to use Kong's API Gateway, however, I want to use its serverless plugin to add some custom implementation before Kong processes the request. I basically want to read in a JSON file volume mapped in the Kong container and do some stuff with it before it processes the request.
When I follow their example I am able to see the logs printed. However, when I start adding Lua code (in custom-auth.lua) it seems to complain. A simple print("hello world") statement on the top of the file gives me the following error.
2021/02/05 02:00:19 [error] 22#0: *8042 [kong] init.lua:270 [pre-function] /usr/local/share/lua/5.1/sandbox.lua:170: [string "..."]:3: attempt to index global 'print' (a nil value), client: 172.31.0.1, server: kong, request: "GET /v1/myEndpoint HTTP/2.0", host: "localhost:8443"
Code reference to error:
170 if not t[1] then error(t[2]) end
sandbox.lua:
local sandbox = {
_VERSION = "sandbox 0.5",
_DESCRIPTION = "A pure-lua solution for running untrusted Lua code.",
_URL = "https://github.com/kikito/sandbox.lua",
_LICENSE = [[
MIT LICENSE
Copyright (c) 2021 Enrique García Cota
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
]],
}
-- quotas don't work in LuaJIT since debug.sethook works differently there
local quota_supported = type(_G.jit) == "nil"
sandbox.quota_supported = quota_supported
-- PUC-Rio Lua 5.1 does not support deactivation of bytecode
local bytecode_blocked = _ENV or type(_G.jit) == "table"
sandbox.bytecode_blocked = bytecode_blocked
-- The base environment is merged with the given env option (or an empty table, if no env provided)
--
local BASE_ENV = {}
-- List of unsafe packages/functions:
--
-- * string.rep: can be used to allocate millions of bytes in 1 operation
-- * {set|get}metatable: can be used to modify the metatable of global objects (strings, integers)
-- * collectgarbage: can affect performance of other systems
-- * dofile: can access the server filesystem
-- * _G: It has access to everything. It can be mocked to other things though.
-- * load{file|string}: All unsafe because they can grant acces to global env
-- * raw{get|set|equal}: Potentially unsafe
-- * module|require|module: Can modify the host settings
-- * string.dump: Can display confidential server info (implementation of functions)
-- * math.randomseed: Can affect the host sytem
-- * io.*, os.*: Most stuff there is unsafe, see below for exceptions
-- Safe packages/functions below
([[
_VERSION assert error ipairs next pairs
pcall select tonumber tostring type unpack xpcall
coroutine.create coroutine.resume coroutine.running coroutine.status
coroutine.wrap coroutine.yield
math.abs math.acos math.asin math.atan math.atan2 math.ceil
math.cos math.cosh math.deg math.exp math.fmod math.floor
math.frexp math.huge math.ldexp math.log math.log10 math.max
math.min math.modf math.pi math.pow math.rad math.random
math.sin math.sinh math.sqrt math.tan math.tanh
os.clock os.difftime os.time
string.byte string.char string.find string.format string.gmatch
string.gsub string.len string.lower string.match string.reverse
string.sub string.upper
table.insert table.maxn table.remove table.sort
]]):gsub('%S+', function(id)
local module, method = id:match('([^%.]+)%.([^%.]+)')
if module then
BASE_ENV[module] = BASE_ENV[module] or {}
BASE_ENV[module][method] = _G[module][method]
else
BASE_ENV[id] = _G[id]
end
end)
local function protect_module(module, module_name)
return setmetatable({}, {
__index = module,
__newindex = function(_, attr_name, _)
error('Can not modify ' .. module_name .. '.' .. attr_name .. '. Protected by the sandbox.')
end
})
end
('coroutine math os string table'):gsub('%S+', function(module_name)
BASE_ENV[module_name] = protect_module(BASE_ENV[module_name], module_name)
end)
-- auxiliary functions/variables
local string_rep = string.rep
local function sethook(f, key, quota)
if type(debug) ~= 'table' or type(debug.sethook) ~= 'function' then return end
debug.sethook(f, key, quota)
end
local function cleanup()
sethook()
string.rep = string_rep -- luacheck: no global
end
-- Public interface: sandbox.protect
function sandbox.protect(code, options)
options = options or {}
local quota = false
if options.quota and not quota_supported then
error("options.quota is not supported on this environment (usually LuaJIT). Please unset options.quota")
end
if options.quota ~= false then
quota = options.quota or 500000
end
assert(type(code) == 'string', "expected a string")
local passed_env = options.env or {}
local env = {}
for k, v in pairs(BASE_ENV) do
local pv = passed_env[k]
if pv ~= nil then
env[k] = pv
else
env[k] = v
end
end
setmetatable(env, { __index = options.env })
env._G = env
local f
if bytecode_blocked then
f = assert(load(code, nil, 't', env))
else
f = assert(loadstring(code))
setfenv(f, env)
end
return function(...)
if quota and quota_supported then
local timeout = function()
cleanup()
error('Quota exceeded: ' .. tostring(quota))
end
sethook(timeout, "", quota)
end
string.rep = nil -- luacheck: no global
local t = table.pack(pcall(f, ...))
cleanup()
if not t[1] then error(t[2]) end
return table.unpack(t, 2, t.n)
end
end
-- Public interface: sandbox.run
function sandbox.run(code, options, ...)
return sandbox.protect(code, options)(...)
end
-- make sandbox(f) == sandbox.protect(f)
setmetatable(sandbox, {__call = function(_,code,o) return sandbox.protect(code,o) end})
return sandbox
Any help would be appreciated.
Figured it out, Kong defaults the Lua modules to sandbox but can be set when starting Kong as an environment variable KONG_UNTRUSTED_LUA: "on" or in the kong.conf to enable all modules.
untrusted_lua = on
# Accepted values are:
#
# - `off`: disallow any loading of Lua functions
# from admin supplied sources (such as via the Admin API).
#
# Note using the `off` option will render plugins such as
# Serverless Functions unusable.
# - `sandbox`: allow loading of Lua functions from admin
# supplied sources, but use a sandbox when
# executing them. The sandboxed
# function will have restricted access
# to the global environment and only
# have access to standard Lua functions
# that will generally not cause harm to
# the Kong node.
#
# In this mode, the `require` function inside
# the sandbox only allows loading external Lua
# modules that are explicitly listed in
# `untrusted_lua_sandbox_requires` below.
#
# LuaJIT bytecode loading is disabled.
#
# Warning: LuaJIT is not designed as a secure
# runtime for running malicious code, therefore,
# you should properly protect your Admin API endpoint
# even with sandboxing enabled. The sandbox only
# provides protection against trivial attackers or
# unintentional modification of the Kong global
# environment.
# - `on`: allow loading of Lua functions from admin
# supplied sources and do not use a sandbox when
# executing them. Functions will have unrestricted
# access to global environment and able to load any
# Lua modules. This is similar to the behavior in Kong
# prior to 2.3.0.
#
# LuaJIT bytecode loading is disabled.
Related
In Lua I can write a simple module like so
local database = require 'database'
local M = {}
function M:GetData()
return database:GetData()
end
return M
Which when required, will load once, and all future versions will load the same copy.
If I wanted to take an object-oriented approach I could do something like:
local M = {}
M.__index = M
function M:GetData()
return self.database:GetData()
end
return function(database)
local newM = setmetatable({}, M)
newM.database = database
return newM
end
Where M is only loaded once, and each copy of newM just holds its own data and uses the methods of the original M.
When it comes to testing, with the OO approach I can just pass in a fake version of 'database' and check it gets called, but with the first approach I can't.
So my question is how can I make the first approach support DI/testing without making it class-like?
My thought was to wrap it in a closure something like this:
local mClosure = function(database)
local M = {}
function M:GetData()
return database:GetData()
end
return M
end
return mClosure
but then every time it is called it will create a new copy of M, so it will lose the benefits of both of the previous approaches.
That's clearly a use case for the Lua debug library. With that you can just modify the upvalues of your function and inject dependancies. Also consider that you can use require for this; just require your database module once, create small table that collects data and then redirects to the original module and put it in package.loaded so the next time you require it, the require call returns the modified version of the module. The OO approach is how you would do this kind of thing in a language like Ruby, but in Lua we have way nicer ways of tapping into a module or function without it being specifically designed for that purpose.
local real_db = require 'db'
local fake_db = setmetatable({}, {__index=db})
function fake_db.exec(query) print('running query: '..query) end -- dummy function
function fake_db.something(...) print('doing something'); real_db.something(...) end
package.loaded.db = fake_db
require 'my_tests' -- this in turn requires 'db', but gets the fake one
package.loaded.db = real_db
-- After this point, `require 'db'` will return the original module
Is there a way to call require in a Lua file, and have the module set the environment of the file that calls it? For example, if I have a DSL (domain specific language) that defines the functions Root and Sequence defined in a table, can I have something like setfenv(1, dslEnv) in the module that allows me to access those functions like global variables?
The goal I in mind is using this is a behavior tree DSL in a way that makes my definition file look like this (or as close it as possible):
require "behaviortrees"
return Root {
Sequence {
Leaf "leafname",
Leaf "leafname"
}
}
without having to specifically bring Root, Sequence, and Leaf into scope explicitly or having to qualify names like behaviortrees.Sequence.
In short, I'm trying to make the definition file as clean as possible, without any extraneous lines cluttering the tree definition.
Can I have something like setfenv(1, dslEnv) in the module that allows me to access those functions like global variables?
Sure you can. You just have to figure out the correct stack level to use instead of the 1 in your setfenv call. Usually you'd walk up the stack using a loop with debug.getinfo calls until you find the require function on the stack, and then you move some more until you find the next main chunk (just in case someone calls require in a function). This is the stack level you'd have to use with setfenv. But may I suggest a ...
Different Approach
require in Lua is pluggable. You can add a function (called a searcher) to the package.loaders array, and require will call it when it tries to load a module. Let's suppose all your DSL files have a .bt suffix instead of the usual .lua. You'd then use a reimplementation of the normal Lua searcher with the differences that you'd look for .bt files instead of .lua files, and that you'd call setfenv on the function returned by loadfile. Something like this:
local function Root( x ) return x end
local function Sequence( x ) return x end
local function Leaf( x ) return x end
local delim = package.config:match( "^(.-)\n" ):gsub( "%%", "%%%%" )
local function searchpath( name, path )
local pname = name:gsub( "%.", delim ):gsub( "%%", "%%%%" )
local msg = {}
for subpath in path:gmatch( "[^;]+" ) do
local fpath = subpath:gsub( "%?", pname ):gsub("%.lua$", ".bt") -- replace suffix
local f = io.open( fpath, "r" )
if f then
f:close()
return fpath
end
msg[ #msg+1 ] = "\n\tno file '"..fpath.."'"
end
return nil, table.concat( msg )
end
local function bt_searcher( modname )
assert( type( modname ) == "string" )
local filename, msg = searchpath( modname, package.path )
if not filename then
return msg
end
local env = { -- create custom environment
Root = Root,
Sequence = Sequence,
Leaf = Leaf,
}
local mod, msg = loadfile( filename )
if not mod then
error( "error loading module '"..modname.."' from file '"..filename..
"':\n\t"..msg, 0 )
end
setfenv( mod, env ) -- set custom environment
return mod, filename
end
table.insert( package.loaders, bt_searcher )
If you put this in a module and require it once from your main program, you can then require your DSL files with the custom environment from .bt files somewhere where you would put your .lua files as well. And you don't even need the require("behaviortrees") in your DSL files. E.g.:
File xxx.bt:
return Root {
Sequence {
Leaf "leafname",
Leaf "leafname"
}
}
File main.lua:
#!/usr/bin/lua5.1
require( "behaviortrees" ) -- loads the Lua module above and adds to package.loaders
print( require( "xxx" ) ) -- loads xxx.bt (but an xxx Lua module would still take precedence)
At least in Lua 5.2, _ENV is a local that determinates the environment table. You can change the environment of any function, basically, the chunk.
_ENV = behaviortrees;
Another way is to automatically copy each field:
do
_ENV = _ENV or _G;
for k, v in next, behaviortrees do
_ENV[k] = v;
end
end
However it might be more efficient to manually local each field from behaviortrees.
Module "behaviortrees.lua"
local behaviortrees = {
-- insert your code for these functions
Root = function(...) ... end,
Sequence = function(...) ... end,
Leaf = function(...) ... end,
}
-- Now set the environment of the caller. Two ways are available:
-- If you want to make DSL environment isolated from Lua globals
-- (for example, "require" and "print" functions will not be available
-- after executing require "behaviortrees")
setfenv(3, behaviortrees)
-- or
-- If you want to preserve all globals for DSL
setfenv(3, setmetatable(behaviortrees, {__index = getfenv(3)}))
Main Lua program:
require "behaviortrees"
return Root {
Sequence {
Leaf "leafname",
Leaf "leafname"
}
}
I have so many records in aerospike, i want to fetch the records whose ttl is -1 please provide solution
Just to clarify, setting a TTL of -1 in the client means never expire (equivalent to a default-ttl of 0 in the server's aerospike.conf file), while setting a TTL of 0 in the client means inherit the default-ttl for this namespace.
With Predicate Filtering:
If you're using the Java, C, C# and Go clients the easiest way to identify the records with a void time of 0 would be to use a predicate filter.
In the Java app:
Statement stmt = new Statement();
stmt.setNamespace(params.namespace);
stmt.setSetName(params.set);
stmt.setPredExp(
PredExp.recVoidTime(),
PredExp.integerValue(0),
PredExp.integerEqual()
);
RecordSet rs = client.query(null, stmt);
Without Predicate Filtering:
With other clients that don't yet have predicate filtering (Python, PHP, etc), you would do it all through a stream UDF. The filtering logic would have to live inside the UDF.
ttl.lua
local function filter_ttl_zero(rec)
local rec_ttl = record.ttl(rec)
if rec_ttl == 0 then
return true
end
return false
end
local function map_record(rec)
local ret = map()
for i, bin_name in ipairs(record.bin_names(rec)) do
ret[bin_name] = rec[bin_name]
end
return ret
end
function get_zero_ttl_recs(stream)
return stream : filter(filter_ttl_zero) : map(map_record)
end
In AQL:
$ aql
Aerospike Query Client
Version 3.12.0
C Client Version 4.1.4
Copyright 2012-2017 Aerospike. All rights reserved.
aql> register module './ttl.lua'
OK, 1 module added.
aql> AGGREGATE ttl.get_zero_ttl_recs() on test.foo
Alternatively, you could run the stream UDF from the client. The following example is for the Python client:
import aerospike
import pprint
config = {'hosts': [('127.0.0.1', 3000)],
'lua': {'system_path':'/usr/local/aerospike/lua/',
'user_path':'/usr/local/aerospike/usr-lua/'}}
client = aerospike.client(config).connect()
pp = pprint.PrettyPrinter(indent=2)
query = client.query('test', 'foo')
query.apply('ttl', 'get_zero_ttl_recs')
records = query.results()
# we expect a dict (map) whose keys are bin names
# each with the associated bin value
pp.pprint(records)
client.close()
Let's say I want a Lua table that will be provided from a third party, not totally reliable, from a file or other IO source.
I get the table as a string, like "{['valid'] = 10}" and I can load it as
externalTable = loadstring("return " .. txtTable)()
But this opens a breach to code injection, ie.: txtTable = os.execute('rm -rf /')
So I did this sanitizing function:
function safeLoadTable(txtTable)
txtTable = tostring(txtTable)
if (string.find(txtTable, "(", 1, true))
then return nil end
local _start = string.find(txtTable, "{", 1, true)
local _end = string.find(string.reverse(txtTable), "}", 1, true)
if (_start == nil or _end == nil)
then return nil end
txtTable = string.sub(txtTable, _start, #txtTable - _end + 1)
print("cropped to ", txtTable)
local pFunc = loadstring("return " .. txtTable)
if (pFunc) then
local _, aTable = pcall(pFunc)
return aTable
end
end
In the worst case it should return nil.
Can this be considered safe against a "regular bad-intentioned person" :)
You could run the unsafe code in a sandbox.
Here is how a simple sandbox could look in Lua 5.1 (error handling omitted for brevity):
local script = [[os.execute("rm -rf /")]]
local env = { print=print, table=table, string=string }
local f, err = loadstring(script)
if err then
-- handle syntax error
end
setfenv(f, env)
local status, err = pcall(f)
if not status then
-- handle runtime error
end
In Lua 5.2 you can load the script into it's own environment using the load function.
The result would be a runtime error returned from pcall:
attempt to index global 'os' (a nil value)
EDIT
As Lorenzo Donati pointed out in the comments this is not a complete solution to stop rogue scripts. It essentially allows you to white-list functions and tables that are approved for user scripts.
For more info about handling rogue scripts I would suggest this SO question:
Embedded Lua - timing out rogue scripts (e.g. infinite loop) - an example anyone?
I don't think it is safe. Try this:
print(safeLoadTable [[{ foo = (function() print"yahoo" end)() } ]])
EDIT
or this, for more fun:
print(safeLoadTable [[{ foo = (function() print(os.getenv "PATH") end)() } ]])
I won't suggest the alternative of replacing that os.getenv with os.execute, though. :-)
The problem is not easy to solve. Code injection avoidance is not at all simple in this case because you are executing a piece of Lua code when doing that loadstring. No simple string matching technique is really safe. The only secure way would be to implement a parser for a subset of the Lua table syntax and use that parser on the string.
BTW, even Lua team stripped off the bytecode verifier from Lua 5.2 since they discovered that it was amenable to attacks, and bytecode is a far simpler language than Lua source code.
I created sandbox.lua for exactly this purpose. It'll handle both insecure stuff as well as DOS-type attacks, assuming that your environment has access to the debug facility.
https://github.com/kikito/sandbox.lua
Note that for now it is Lua 5.1-compatible only.
Running in sandbox isn't safe, inspecting source code is not very simple. An idea: inspect bytecode!
Emmm, actually that's not very simple either, but here is a lazy implementation: http://codepad.org/mGqQ0Y8q
How can I recreate the functionality of setfenv in Lua 5.2? I'm having some trouble understanding exactly how you are supposed to use the new _ENV environment variable.
In Lua 5.1 you can use setfenv to sandbox any function quite easily.
--# Lua 5.1
print('_G', _G) -- address of _G
local foo = function()
print('env', _G) -- address of sandbox _G
bar = 1
end
-- create a simple sandbox
local env = { print = print }
env._G = env
-- set the environment and call the function
setfenv(foo, env)
foo()
-- we should have global in our environment table but not in _G
print(bar, env.bar)
Running this example shows an output:
_G table: 0x62d6b0
env table: 0x635d00
nil 1
I would like to recreate this simple example in Lua 5.2. Below is my attempt, but it does not work like the above example.
--# Lua 5.2
local function setfenv(f, env)
local _ENV = env or {} -- create the _ENV upvalue
return function(...)
print('upvalue', _ENV) -- address of _ENV upvalue
return f(...)
end
end
local foo = function()
print('_ENV', _ENV) -- address of function _ENV
bar = 1
end
-- create a simple sandbox
local env = { print = print }
env._G = env
-- set the environment and call the function
foo_env = setfenv(foo, env)
foo_env()
-- we should have global in our envoirnment table but not in _G
print(bar, env.bar)
Running this example shows the output:
upvalue table: 0x637e90
_ENV table: 0x6305f0
1 nil
I am aware of several other questions on this subject, but they mostly seem to be dealing with loading dynamic code (files or string) which work quite well using the new load function provided in Lua 5.2. Here I am specifically asking for a solution to run arbitrary functions in a sandbox. I would like to do this without using the debug library. According to the Lua documentation we should not have to rely on it.
You cannot change the environment of a function without using the debug library from Lua in Lua 5.2. Once a function has been created, that is the environment it has. The only way to modify this environment is by modifying its first upvalue, which requires the debug library.
The general idea with environments in Lua 5.2 is that the environment should be considered immutable outside of trickery (ie: the debug library). You create a function in an environment; once created there, that's the environment it has. Forever.
This is how environments were often used in Lua 5.1, but it was easy and sanctioned to modify the environment of anything with a casual function call. And if your Lua interpreter removed setfenv (to prevent users from breaking the sandbox), then the user code can't set the environment for their own functions internally. So the outside world gets a sandbox, but the inside world can't have a sandbox within the sandbox.
The Lua 5.2 mechanism makes it harder to modify the environment post function-creation, but it does allow you to set the environment during creation. Which lets you sandbox inside the sandbox.
So what you really want is to just rearrange your code like this:
local foo;
do
local _ENV = { print = print }
function foo()
print('env', _ENV)
bar = 1
end
end
foo is now sandboxed. And now, it's much harder for someone to break the sandbox.
As you can imagine, this has caused some contention among Lua developers.
It's a bit expensive, but if it's that important to you...
Why not use string.dump, and re-load the function into the right environment?
function setfenv(f, env)
return load(string.dump(f), nil, nil, env)
end
function foo()
herp(derp)
end
setfenv(foo, {herp = print, derp = "Hello, world!"})()
To recreate setfenv/getfenv in Lua 5.2 you can do the following:
if not setfenv then -- Lua 5.2
-- based on http://lua-users.org/lists/lua-l/2010-06/msg00314.html
-- this assumes f is a function
local function findenv(f)
local level = 1
repeat
local name, value = debug.getupvalue(f, level)
if name == '_ENV' then return level, value end
level = level + 1
until name == nil
return nil end
getfenv = function (f) return(select(2, findenv(f)) or _G) end
setfenv = function (f, t)
local level = findenv(f)
if level then debug.setupvalue(f, level, t) end
return f end
end
RPFeltz's answer (load(string.dump(f)...)) is a clever one and may work for you, but it doesn't deal with functions that have upvalues (other than _ENV).
There is also compat-env module that implements Lua 5.1 functions in Lua 5.2 and vice versa.
In Lua5.2 a sandboxeable function needs to specify that itself. One simple pattern you can use is have it receive _ENV as an argument
function(_ENV)
...
end
Or wrap it inside something that defines the env
local mk_func(_ENV)
return function()
...
end
end
local f = mk_func({print = print})
However, this explicit use of _ENV is less useful for sandboxing, since you can't always assume the other function will cooperate by having an _ENV variable. In that case, it depends on what you do. If you just want to load code from some other file then functions such as load and loadfile usually receive an optional environment parameter that you can use for sandboxing. Additionally, if the code you are trying to load is in string format you can use string manipulation to add _ENV variables yourself (say, by wrapping a function with an env parameter around it)
local code = 'return function(_ENV) return ' .. their_code .. 'end'
Finally, if you really need dynamic function environment manipulation, you can use the debug library to change the function's internal upvalue for _ENV. While using the debug library is not usually encouraged, I think it is acceptable if all the other alternatives didn't apply (I feel that in this case changing the function's environment is deep voodoo magic already so using the debug library is not much worse)