syncmaildir 1.2.2-1ubuntu1 / usr / bin / smd-client

#! /usr/bin/env lua5.1
-- 
-- Released under the terms of GPLv3 or at your option any later version.
-- No warranties.
-- Copyright Enrico Tassi <gares@fettunta.org>

require 'syncmaildir'

-- export syncmaildir to the global namespace
for k,v in pairs(syncmaildir) do _G[k] = v end

-- globals counter for statistics
local statistics = {
	added = 0,
	removed = 0,
	received = 0,
	xdelta = 0,
}

-- ========================= get mail queue =================================
-- queue for fetching mails in blocks of queue_max_len messages
-- to cope with latency

local get_full_email_queue = {}
local queue_max_len = 50

function process_get_full_email_queue()
	local command = {}
	for _,v in ipairs(get_full_email_queue) do
		command[#command+1] = 'GET ' .. v.name
	end
	command[#command+1] = ''
	io.write(table.concat(command,'\n'))
	command = nil
	io.flush()
	local tmp = {}
	for _,v in ipairs(get_full_email_queue) do
		tmp[#tmp+1] = tmp_for(v.name)
		v.tmp = tmp[#tmp]
		statistics.received = statistics.received + receive(io.stdin, tmp[#tmp])
	end
	tmp = nil
	for _,v in ipairs(get_full_email_queue) do
		local hsha_l, bsha_l = sha_file(v.tmp)
		if hsha_l == v.hsha and bsha_l == v.bsha then
			local rc = os.rename(v.tmp, v.name) 
			if rc then
				statistics.added = statistics.added + 1
			else
				log_error('Failed to rename '..v.tmp..' to '..v.name)
				log_error('It may be caused by bad directory permissions, '..
					'please check.')
				os.remove(v.tmp)
				return (trace(false)) -- fail rename tmpfile to actual name
			end
		else
			log_error('The server sent a different email for '..v.name)
			log_error('This problem should be transient, please retry.')
			os.remove(v.tmp)
			return (trace(false)) -- get full email failed, received wrong mail
		end
	end
	get_full_email_queue = {}
	return (trace(true)) -- get full email OK
end

function process_pending_queue()
	local rc = process_get_full_email_queue()
	if not rc then
		io.write('ABORT\n')
		io.flush()
		os.exit(1)
	end
end

-- the function to fetch a mail message
function get_full_email(name,hsha,bsha)
	if dry_run() then
		statistics.added = statistics.added + 1
		return true
	end
	get_full_email_queue[#get_full_email_queue+1] = {
		name = name;
		hsha = hsha;
		bsha = bsha;
	}
	return true
end

-- ======================== header replacing =================================

function merge_mail(header,body,target)
	local h = io.open(header,"r")
	local b = io.open(body,"r")
	local t = io.open(target,"w")
	local l
	while true do
		l = h:read("*l")
		if l and l ~= "" then t:write(l,'\n') else break end
	end
	while true do
		l = b:read("*l")
		if not l or l == "" then break end
	end
	t:write('\n')
	while true do
		l = b:read("*l")
		if l then t:write(l,'\n') else break end
	end
	h:close()
	b:close()
	t:close()
end

function get_header_and_merge(name,hsha)
	local tmpfile = tmp_for(name)
	io.write('GETHEADER '..name..'\n')
	io.flush()
	receive(io.stdin, tmpfile)
	local hsha_l, _ = sha_file(tmpfile)
	if hsha_l == hsha then
		if not dry_run() then
			local tmpfile1 = tmp_for(name)
			merge_mail(tmpfile,name,tmpfile1)
			os.remove(tmpfile)
			os.rename(tmpfile1, name)
		else
			-- we delete the new piece without merging (--dry-run)	
			os.remove(tmpfile)
		end
		return (trace(true)) -- get header OK
	else
		os.remove(tmpfile)
		log_error('The server sent a different email header for '..name)
		log_error('This problem should be transient, please retry.')

		log_tags("receive-header","modify-while-update",false,"retry")
		return (trace(false)) -- get header fails, got a different header
	end
end

-- ============================= actions =====================================

function execute_add(name, hsha, bsha)
	local ex, hsha_l, bsha_l = exists_and_sha(name)
	if ex then
		if hsha == hsha_l and bsha == bsha_l then
			return (trace(true)) -- skipping add since already there
		else
			log_error('Failed to add '..name..
				' since a file with the same name')
			log_error('exists but its content is different.')
			log_error('To fix this problem you should rename '..name)
			log_error('Executing `cd; mv -n '..quote(name)..' '..
				quote(tmp_for(name,false))..'` should work.')

			log_tags("mail-addition","concurrent-mailbox-edit",true,
				mk_act("mv", name))
			return (trace(false)) -- skipping add since already there but !=
		end
	end
	return (get_full_email(name,hsha,bsha))
end

function execute_delete(name, hsha, bsha)
	local ex, hsha_l, bsha_l = exists_and_sha(name)
	if ex then
		if hsha == hsha_l and bsha == bsha_l then
			local rc
			if not dry_run() then
				rc = os.remove(name)
			else
				rc = true -- we do not delete the message for real (--dry-run)
			end
			if rc then
				statistics.removed = statistics.removed + 1
				return (trace(true)) -- removed successfully
			else
				log_error('Deletion of '..name..' failed.')
				log_error('It may be caused by bad directory permissions, '..
					'please check.')

				log_tags("delete-message","bad-directory-permission",true,
					mk_act("permission",name))
				return (trace(false)) -- os.remove failed
			end
		else
			log_error('Failed to delete '..name..
				' since the local copy of it has')
			log_error('modifications.')
			log_error('To fix this problem you have two options:')
			log_error('- delete '..name..' by hand')
			log_error('- run @@INVERSECOMMAND@@ so that this file is added '..
				'to the other mailbox')

			log_tags("delete-message", "concurrent-mailbox-edit",true,
				mk_act('display',name),
				mk_act('rm',name),
				"run(@@INVERSECOMMAND@@ @@ENDPOINT@@)")
			return (trace(false)) -- remove fails since local file is !=
		end
	end
	return (trace(true)) -- already removed
end

function execute_copy(name_src, hsha, bsha, name_tgt)
	local ex_src, hsha_src, bsha_src = exists_and_sha(name_src)
	local ex_tgt, hsha_tgt, bsha_tgt = exists_and_sha(name_tgt)
	if ex_src and ex_tgt then
		if hsha_src == hsha_tgt and bsha_src == bsha_tgt then
			return (trace(true)) -- skip copy, already there
		else
			log_error('Failed to copy '..name_src..' to '..name_tgt)
			log_error('The destination already exists but its content differs.')
			log_error('To fix this problem you have two options:')
			log_error('- rename '..name_tgt..' by hand so that '..name_src)
			log_error('  can be copied without replacing it.')
			log_error('  Executing `cd; mv -n '..quote(name_tgt)..' '..
				quote(tmp_for(name_tgt,false))..'` should work.')
			log_error('- run @@INVERSECOMMAND@@ so that your changes to '..
				name_tgt)
			log_error('  are propagated to the other mailbox')

			log_tags("copy-message","concurrent-mailbox-edit",true,
				mk_act('mv',name_tgt),
				"run(@@INVERSECOMMAND@@ @@ENDPOINT@@)")
			return (trace(false)) -- fail copy, already there but !=
		end
	elseif ex_src and not ex_tgt then
		if hsha_src == hsha and bsha_src == bsha then
				local ok, err
				if not dry_run() then
					ok, err = cp(name_src,name_tgt)
				else
					ok = 0 -- we do not copy for real (--dry-run)
				end
				if ok == 0 then 
					return (trace(true)) -- copy successful
				else 
					log_error('Failed to copy '..name_src..' to '..name_tgt..
						' : '..(err or 'unknown error'))

					log_tags("delete-message","bad-directory-permission",true,
						mk_act('display', name_tgt))
					return (trace(false)) -- copy failed (cp command failed)
				end
		else
				-- sub-optimal, we may reuse body or header 
				return (get_full_email(name_tgt,hsha,bsha))
		end
	elseif not ex_src and ex_tgt then
		if hsha == hsha_tgt and bsha == bsha_tgt then
			return (trace(true)) -- skip copy, already there (only the copy)
		else
			log_error('Failed to copy '..name_src..' to '..name_tgt)
			log_error('The source file has been locally removed.')
			log_error('The destination file already exists but its '..
				'content differs.')
			log_error('To fix this problem you have two options:')
			log_error('- rename '..name_tgt..' by hand so that '..
				name_src..' can be')
			log_error('  copied without replacing it.')
			log_error('  Executing `cd; mv -n '..quote(name_tgt)..' '..
				quote(tmp_for(name_tgt,false))..'` should work.')
			log_error('- run @@INVERSECOMMAND@@ so that your changes to '..
				name_tgt..' are')
			log_error('  propagated to the other mailbox')

			log_tags("copy-message","concurrent-mailbox-edit",true,
				mk_act('mv', name_tgt),
				"run(@@INVERSECOMMAND@@ @@ENDPOINT@@)")
			return (trace(false)) -- skip copy, already there and !=, no source
		end
	else
		return (get_full_email(name_tgt,hsha,bsha))
	end
end

function execute_replaceheader(name, hsha, bsha, hsha_new)
	if exists(name) then
		local hsha_l, bsha_l = sha_file(name)
		if hsha == hsha_l and bsha == bsha_l then
			return (get_header_and_merge(name,hsha_new))
		elseif hsha_l == hsha_new and bsha == bsha_l then
			return (trace(true)) -- replace header ok, already changend
		else
			log_error('Failed to replace '..name..' header since it has local')
			log_error(' modifications.')
			log_error('To fix this problem you should rename '..name)
			log_error('Executing `cd; mv -n '..quote(name)..' '..
				quote(tmp_for(name,false))..'` should work.')
			log_tags("header-replacement","concurrent-mailbox-edit",true,
				mk_act('mv', name))
			return (trace(false)) -- replace header fails, local header !=
		end
	else
		return (get_full_email(name,hsha_new,bsha))
	end
end

function execute_copybody(name, bsha, newname, hsha)
	local exn, hsha_ln, bsha_ln = exists_and_sha(newname)
	if not exn then
		local ex, _, bsha_l = exists_and_sha(name)
		if ex and bsha_l == bsha then
			local ok, err
			if not dry_run() then
				ok, err = cp(name,newname)
			else
				ok = 0 -- we do not copy the body for merging (--dry-run)
			end
			if ok == 0 then 
				ok = get_header_and_merge(newname,hsha)
				if ok then
					return (trace(true)) -- copybody OK
				else
					os.remove(newname)
					return (trace(false)) -- copybody failed, bad new header
				end
			else 
				log_error('Failed to copy '..name..' to '..newname..' : '..
					(err or 'unknown error'))
					
				log_tags("copy-message","bad-directory-permission",true,
					mk_act('display', newname))
				return (trace(false)) -- copybody failed (cp command failed)
			end
		else
			return(get_full_email(newname,hsha,bsha))
		end
	else
		if bsha == bsha_ln and hsha == hsha_ln then
			return (trace(true)) -- copybody OK (already there)
		else
			log_error('Failed to copy body of '..name..' to '..newname)
			log_error('To fix this problem you should rename '..newname)
			log_error('Executing `cd; mv -n '..quote(newname)..' '..
				quote(tmp_for(newname,false))..'` should work.')

			log_tags("copy-body","concurrent-mailbox-edit",true,
				mk_act('mv', newname))
			return (trace(false)) -- copybody failed (already there, != )
		end
	end
end

function execute_replace(name1, hsha1, bsha1, hsha2, bsha2)
	local exn, hsha_ln, bsha_ln = exists_and_sha(name1)
	if not exn then
		return(get_full_email(name1,hsha2,bsha2))
	else
		if bsha2 == bsha_ln and hsha2 == hsha_ln then
			return (trace(true)) -- replace OK (already there)
		elseif bsha1 == bsha_ln and hsha1 == hsha_ln then
			return(get_full_email(name1,hsha2,bsha2))
		else
			log_error('Failed to replace '..name1)
			log_error('To fix this problem you should rename '..name1)
			log_error('Executing `cd; mv -n '..quote(name1)..' '..
				quote(tmp_for(name1,false))..'` should work.')

			log_tags("replace","concurrent-mailbox-edit",true,
				mk_act('mv', name1))
			return (trace(false)) -- replace failed (already there, != )
		end
	end
end

function execute_error(msg)
	
	log_error('mddiff failed: '..msg)
	if msg:match('^Unable to open directory') then
		log_tags("mddiff","directory-disappeared",false)
	else
		log_tags("mddiff","unknown",true)
	end

	return (trace(false)) -- mddiff error
end

-- the main switch, dispatching actions.
-- extra parentheses around execute_* calls make it a non tail call,
-- thus we get the stack frame print in case of error.
function execute(cmd)
	local opcode = parse(cmd, '^(%S+)')

	if opcode == "ADD" then 
		local name, hsha, bsha = parse(cmd, '^ADD (%S+) (%S+) (%S+)$')
		name = url_decode(name)
		mkdir_p(name)
		return (execute_add(name, hsha, bsha))
	end

	if opcode == "DELETE" then 
		local name, hsha, bsha = parse(cmd, '^DELETE (%S+) (%S+) (%S+)$')
		name = url_decode(name)
		mkdir_p(name)
		return (execute_delete(name, hsha, bsha))
	end

	if opcode == "COPY" then 
		local name_src, hsha, bsha, name_tgt = 
			parse(cmd, '^COPY (%S+) (%S+) (%S+) TO (%S+)$')
		name_src = url_decode(name_src)
		name_tgt = url_decode(name_tgt)
		mkdir_p(name_src)
		mkdir_p(name_tgt)
		return (execute_copy(name_src, hsha, bsha, name_tgt))
	end
	
	if opcode == "REPLACEHEADER" then
		local name, hsha, bsha, hsha_new = 
			parse(cmd, '^REPLACEHEADER (%S+) (%S+) (%S+) WITH (%S+)$')
		name = url_decode(name)
		mkdir_p(name)
		return (execute_replaceheader(name, hsha, bsha, hsha_new))
	end

	if opcode == "COPYBODY" then
		local name, bsha, newname, hsha = 
			parse(cmd, '^COPYBODY (%S+) (%S+) TO (%S+) (%S+)$')
		name = url_decode(name)
		newname = url_decode(newname)
		mkdir_p(name)
		mkdir_p(newname)
		return (execute_copybody(name, bsha, newname, hsha))
	end

	if opcode == "REPLACE" then
		local name1, hsha1, bsha1, hsha2, bsha2 = 
			parse(cmd, '^REPLACE (%S+) (%S+) (%S+) WITH (%S+) (%S+)$')
		name1 = url_decode(name1)
		mkdir_p(name1)
		return (execute_replace(name1, hsha1, bsha1, hsha2, bsha2))
	end

	if opcode == "ERROR" then
		local msg = parse(cmd, '^ERROR (.*)$')
		return (execute_error(msg))
	end
	
	log_internal_error_and_fail('Unknown opcode '..opcode, "protocol")
end

-- ============================= MAIN =====================================

-- receive a list of commands
function receive_delta(inf)
	local cmds = {}
	local line = ""

	repeat
		line = inf:read("*l")
		if line and line ~= "END" then cmds[#cmds+1] = line end
	until not line or line == "END"
	if line ~= "END" then
		log_error('Unable to receive a complete diff')
		log_tags_and_fail("network error while receiving delta",
			"receive-delta","network",false,"retry")
	end

	return cmds
end

function main()
	-- sanity checks for external softwares
	assert_exists(MDDIFF)
	assert_exists(XDELTA)

	-- argument parsing
	local usage = "Usage: "..arg[0]:match('[^/]+$')..
		" [-vd] [-t translatorRL] endpointname mailboxes...\n"
	while #arg > 2 do
		if arg[1] == '-v' or arg[1] == '--verbose' then
			set_verbose(true)
			table.remove(arg,1)
		elseif arg[1] == '-d' or arg[1] == '--dry-run' then
			set_dry_run(true)
			table.remove(arg,1)
		elseif arg[1] == '-t' or arg[1] == '--translator' then
			set_translator(arg[2])
			table.remove(arg,1)
			table.remove(arg,1)
		else
			break
		end
	end
	
	if #arg < 2 then
		io.stderr:write(usage)
		os.exit(2)
	end

	-- here we go
	local endpoint = arg[1]
	table.remove(arg,1)
	local dbfile = dbfile_name(endpoint, arg)
	local xdelta = dbfile .. '.xdelta'
	local newdb = dbfile .. '.new'
	
	-- sanity check, translator and absolute paths cannot work
	for _, v in ipairs(arg) do
		if v:byte(1) == string.byte('/',1) then
			log_error("Absolute paths are not supported: "..v)
			log_tags_and_fail("Absolute path detected",
				"main","mailbox-has--absolute-path",true)
		end
	end

	-- we check the protocol version and dbfile fingerprint
	handshake(dbfile)
	
	-- receive and process commands
	local commands = receive_delta(io.stdin)
	for i,cmd in ipairs(commands) do
		local rc = execute(cmd)
		if not rc then
			io.write('ABORT\n')
			io.flush()
			os.exit(3)
		end
		-- some commands are delayed, we fire them in block
		if #get_full_email_queue > queue_max_len then
				process_pending_queue()
		end
	end
	-- some commands may still be in the queue, we fire them now
	process_pending_queue()
	
	-- we commit and update the dbfile
	io.write('COMMIT\n')
	io.flush()
	statistics.xdelta = receive(io.stdin, xdelta)

	local rc
	if not dry_run() then 
		rc = os.execute(XDELTA..' patch '..xdelta..' '..dbfile..' '..newdb)
	else
		rc = 0 -- the xdelta transmitted with --dry-run is dummy
	end
	if rc ~= 0 and rc ~= 256 then
		log_error('Unable to apply delta to dbfile.')
		io.write('ABORT\n')
		io.flush()
		os.exit(4)
	end
	if not dry_run() then
		rc = os.rename(newdb,dbfile)
	else
		rc = true -- with --dry-run there is no xdelta affair
	end
	if not rc then
		log_error('Unable to rename '..newdb..' to '..dbfile)
		io.write('ABORT\n')
		io.flush()
		os.exit(5)
	end
	os.remove(xdelta)
	io.write('DONE\n')
	io.flush()

	-- some machine understandable output before quitting
	log_tag('stats::new-mails('..  statistics.added..
		'), del-mails('..statistics.removed..
		'), bytes-received('..statistics.received..
		'), xdelta-received('..statistics.xdelta..
		')')

	os.exit(0)
end

-- no more global variables
set_strict()

-- parachute for errors
parachute(main, 6)

-- vim:set ts=4: