--[[--------------------------------------------------------------------

  llex.lua: Lua 5.1 lexical analyzer in Lua
  This file is part of LuaSrcDiet, based on Yueliang material.

  Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net>
  The COPYRIGHT file describes the conditions
  under which this software may be distributed.

  See the ChangeLog for more information.

------------------------------------------------------------------------]]

--[[--------------------------------------------------------------------
-- NOTES:
-- * This is a version of the native 5.1.x lexer from Yueliang 0.4.0,
--   with significant modifications to handle LuaSrcDiet's needs:
--   (1) llex.error is an optional error function handler
--   (2) seminfo for strings include their delimiters and no
--       translation operations are performed on them
-- * ADDED shbang handling has been added to support executable scripts
-- * NO localized decimal point replacement magic
-- * NO limit to number of lines
-- * NO support for compatible long strings (LUA_COMPAT_LSTR)
-- * Please read technotes.txt for more technical details.
------------------------------------------------------------------------]]

local base=_G
local string=require"string"
module"llex"

local find=string.find
local match=string.match
local sub=string.sub

----------------------------------------------------------------------
-- initialize keyword list, variables
----------------------------------------------------------------------

local kw={}
for v in string.gmatch([[
and break do else elseif end false for function if in
local nil not or repeat return then true until while]],"%S+")do
kw[v]=true
end

-- NOTE: see init() for module variables (externally visible):
--       tok, seminfo, tokln

local z,-- source stream
sourceid,-- name of source
I,-- position of lexer
buff,-- buffer for strings
ln-- line number

----------------------------------------------------------------------
-- add information to token listing
----------------------------------------------------------------------

local function addtoken(token,info)
local i=#tok+1
tok[i]=token
seminfo[i]=info
tokln[i]=ln
end

----------------------------------------------------------------------
-- handles line number incrementation and end-of-line characters
----------------------------------------------------------------------

local function inclinenumber(i,is_tok)
local sub=sub
local old=sub(z,i,i)
i=i+1-- skip '\n' or '\r'
local c=sub(z,i,i)
if(c=="\n"or c=="\r")and(c~=old)then
i=i+1-- skip '\n\r' or '\r\n'
old=old..c
end
if is_tok then addtoken("TK_EOL",old)end
ln=ln+1
I=i
return i
end

----------------------------------------------------------------------
-- initialize lexer for given source _z and source name _sourceid
----------------------------------------------------------------------

function init(_z,_sourceid)
z=_z-- source
sourceid=_sourceid-- name of source
I=1-- lexer's position in source
ln=1-- line number
tok={}-- lexed token list*
seminfo={}-- lexed semantic information list*
tokln={}-- line numbers for messages*
-- (*) externally visible thru' module
--------------------------------------------------------------------
-- initial processing (shbang handling)
--------------------------------------------------------------------
local p,_,q,r=find(z,"^(#[^\r\n]*)(\r?\n?)")
if p then-- skip first line
I=I+#q
addtoken("TK_COMMENT",q)
if#r>0 then inclinenumber(I,true)end
end
end

----------------------------------------------------------------------
-- returns a chunk name or id, no truncation for long names
----------------------------------------------------------------------

function chunkid()
if sourceid and match(sourceid,"^[=@]")then
return sub(sourceid,2)-- remove first char
end
return"[string]"
end

----------------------------------------------------------------------
-- formats error message and throws error
-- * a simplified version, does not report what token was responsible
----------------------------------------------------------------------

function errorline(s,line)
local e=error or base.error
e(string.format("%s:%d: %s",chunkid(),line or ln,s))
end

------------------------------------------------------------------------
-- count separators ("=") in a long string delimiter
------------------------------------------------------------------------

local function skip_sep(i)
local sub=sub
local s=sub(z,i,i)
i=i+1
local count=#match(z,"=*",i)-- note, take the length
i=i+count
I=i
return(sub(z,i,i)==s)and count or(-count)-1
end

----------------------------------------------------------------------
-- reads a long string or long comment
----------------------------------------------------------------------

local function read_long_string(is_str,sep)
local i=I+1-- skip 2nd '['
local sub=sub
local c=sub(z,i,i)
if c=="\r"or c=="\n"then-- string starts with a newline?
i=inclinenumber(i)-- skip it
end
local j=i
while true do
local p,q,r=find(z,"([\r\n%]])",i)-- (long range)
if not p then
errorline(is_str and"unfinished long string"or
"unfinished long comment")
end
i=p
if r=="]"then-- delimiter test
if skip_sep(i)==sep then
buff=sub(z,buff,I)
I=I+1-- skip 2nd ']'
return buff
end
i=I
else-- newline
buff=buff.."\n"
i=inclinenumber(i)
end
end--while
end

----------------------------------------------------------------------
-- reads a string
----------------------------------------------------------------------

local function read_string(del)
local i=I
local find=find
local sub=sub
while true do
local p,q,r=find(z,"([\n\r\\\"\'])",i)-- (long range)
if p then
if r=="\n"or r=="\r"then
errorline("unfinished string")
end
i=p
if r=="\\"then-- handle escapes
i=i+1
r=sub(z,i,i)
if r==""then break end-- (EOZ error)
p=find("abfnrtv\n\r",r,1,true)
------------------------------------------------------
if p then-- special escapes
if p>7 then
i=inclinenumber(i)
else
i=i+1
end
------------------------------------------------------
elseif find(r,"%D")then-- other non-digits
i=i+1
------------------------------------------------------
else-- \xxx sequence
local p,q,s=find(z,"^(%d%d?%d?)",i)
i=q+1
if s+1>256 then-- UCHAR_MAX
errorline("escape sequence too large")
end
------------------------------------------------------
end--if p
else
i=i+1
if r==del then-- ending delimiter
I=i
return sub(z,buff,i-1)-- return string
end
end--if r
else
break-- (error)
end--if p
end--while
errorline("unfinished string")
end

------------------------------------------------------------------------
-- main lexer function
------------------------------------------------------------------------

function llex()
local find=find
local match=match
while true do--outer
local i=I
-- inner loop allows break to be used to nicely section tests
while true do--inner
----------------------------------------------------------------
local p,_,r=find(z,"^([_%a][_%w]*)",i)
if p then
I=i+#r
if kw[r]then
addtoken("TK_KEYWORD",r)-- reserved word (keyword)
else
addtoken("TK_NAME",r)-- identifier
end
break-- (continue)
end
----------------------------------------------------------------
local p,_,r=find(z,"^(%.?)%d",i)
if p then-- numeral
if r=="."then i=i+1 end
local _,q,r=find(z,"^%d*[%.%d]*([eE]?)",i)
i=q+1
if#r==1 then-- optional exponent
if match(z,"^[%+%-]",i)then-- optional sign
i=i+1
end
end
local _,q=find(z,"^[_%w]*",i)
I=q+1
local v=sub(z,p,q)-- string equivalent
if not base.tonumber(v)then-- handles hex test also
errorline("malformed number")
end
addtoken("TK_NUMBER",v)
break-- (continue)
end
----------------------------------------------------------------
local p,q,r,t=find(z,"^((%s)[ \t\v\f]*)",i)
if p then
if t=="\n"or t=="\r"then-- newline
inclinenumber(i,true)
else
I=q+1-- whitespace
addtoken("TK_SPACE",r)
end
break-- (continue)
end
----------------------------------------------------------------
local r=match(z,"^%p",i)
if r then
buff=i
local p=find("-[\"\'.=<>~",r,1,true)
if p then
-- two-level if block for punctuation/symbols
--------------------------------------------------------
if p<=2 then
if p==1 then-- minus
local c=match(z,"^%-%-(%[?)",i)
if c then
i=i+2
local sep=-1
if c=="["then
sep=skip_sep(i)
end
if sep>=0 then-- long comment
addtoken("TK_LCOMMENT",read_long_string(false,sep))
else-- short comment
I=find(z,"[\n\r]",i)or(#z+1)
addtoken("TK_COMMENT",sub(z,buff,I-1))
end
break-- (continue)
end
-- (fall through for "-")
else-- [ or long string
local sep=skip_sep(i)
if sep>=0 then
addtoken("TK_LSTRING",read_long_string(true,sep))
elseif sep==-1 then
addtoken("TK_OP","[")
else
errorline("invalid long string delimiter")
end
break-- (continue)
end
--------------------------------------------------------
elseif p<=5 then
if p<5 then-- strings
I=i+1
addtoken("TK_STRING",read_string(r))
break-- (continue)
end
r=match(z,"^%.%.?%.?",i)-- .|..|... dots
-- (fall through)
--------------------------------------------------------
else-- relational
r=match(z,"^%p=?",i)
-- (fall through)
end
end
I=i+#r
addtoken("TK_OP",r)-- for other symbols, fall through
break-- (continue)
end
----------------------------------------------------------------
local r=sub(z,i,i)
if r~=""then
I=i+1
addtoken("TK_OP",r)-- other single-char tokens
break
end
addtoken("TK_EOS","")-- end of stream,
return-- exit here
----------------------------------------------------------------
end--while inner
end--while outer
end

return base.getfenv()