falconpl-dev 0.9.6.9-git20120606-2.1+b1 / usr / bin / fallc.fal

#!/usr/bin/env falcon
/***********************************************************
* Falcon international translation compiler
*
* See LICENSE file for licensing details.
************************************************************/

directive version=0x010100, lang="en_US"
load mxml
load regex


//=============================================
// Little class for simpler regex
//
class XRegex( expr, pcount ) from Regex(expr)
   pcount = pcount

   function replaceAll( str, chg )
      try
         return self.Regex.replaceAll( str, chg )
      catch RegexError
         > @i"fallc: Warning, cannot replace variables from $(str)"
         return str
      end
   end

   function grabAll( str )
      s = self.pcount
      try
         return map( {part => str[part[0] + s : part[1]]},
               self.findAll( str )  )
      catch RegexError
         > @i"fallc: Warning, cannot grab variables from $(str)"
         return []
      end
   end
end

reVars = XRegex( '\$([^+*/;:()|^!@#[\]\s?$-]+)', 1 )
reParVars = XRegex( '\$\(([^+*/;:()|^!@#?$-]+)', 2 )
reMover = XRegex( '\s+' )

object Options from CmdlineParser
   output = nil
   inputs = []
   globs = []
   checkvars = true
   merge = nil

   function onOption( opt )
      switch opt
         case 'o', 'm'
            self.expectValue()

         case 'h': usage()
         case 'v': version()
         case 'c': self.checkvars = false

         default
            usage()
            exit(1)
      end
   end

   function onValue( opt, val )
      switch opt
         case 'o': self.output = val
         case 'm': self.merge = val
      end
   end

   function onFree( opt )
      if "*" in opt or "?" in opt
         self.globs += opt
      else
         self.inputs += opt
      end
   end
end

function usage()
   version()
   >
   > i'
   Usage:
      fallc [options] files ...
     
   Options:
      -c            Do NOT check for consistency in \$(varname) blocks.
      -m <file>     Merge mode (get original file .ftt and insert translation.
      -o <output>   Writes the results to this file.
      -h            Prints this help.
      -v            Prints version informations.
'
end

function version()
   ver = vmModuleVersionInfo()
   > @"Falcon Language Tables Compiler, version $(ver[0]).$(ver[1]).$(ver[2])"
end

//====================================================
// Class storing a filewide language table
//
class LanguageTable( fromLang, tglang, file, module )
   fromLang = fromLang
   name = tglang
   file = file
   modName = module
   entries = [=>]
   originals = [=>]
end

//====================================================
// Parsing a language table
//
function parseLanguage( file )

   try
      // read the XML document
      doc = MXMLDocument( "utf-8" )
      doc.read( file )
      // now get the root's node child, if any
      root = doc.root()
      if not root or root.name() != "translation"
         > @i"fallc: Warning: root node for \"$(file)\" is not \"translation\". Ignoring."
         return nil
      end

      attr = root.getAttribs()
      if "module" notin attr or "into" notin attr or "from" notin attr
         > @i"fallc: Warning: missing informations in root node in \"$(file)\". Ignoring."
         return nil
      end

      //Ok, we can start iterating.
      ltab = LanguageTable( attr["from"], attr["into"], file, attr["module"] )
      child = root.firstChild()
      while child != nil

         if child.name() !="string"
            child = child.nextSibling()
            continue
         end

         attr = child.getAttribs()
         if "id" notin attr
            child = child.nextSibling()
            continue
         end

         try
            stringId = int( attr["id"] )
         catch
            child = child.nextSibling()
            continue
         end

         // now get the original and translated subnodes.
         chchild = child.firstChild()
         translated = original = nil

         while chchild != nil
            if chchild.name() == "original"
               original = chchild.data()
            elif chchild.name() == "translated"
               translated = chchild.data()
            end

            chchild = chchild.nextSibling()
         end

         // warn if the transalted string has not the same variables as the original
         if Options.checkvars
            origVars = reVars.grabAll( original ) + reParVars.grabAll( original )
            transVars = reVars.grabAll( translated ) + reParVars.grabAll( translated )

            //normalize spaces
            origVars = map( {x => reMover.replaceAll( x, "" )} , origVars )
            transVars = map( {x => reMover.replaceAll( x, "" )}, transVars )

            for var in origVars
               if var notin transVars
                  > @i"fallc: Warning: variable '$(var)' in string ID=$(stringId) has not been translated"
               end
            end

            for var in transVars
               if var notin origVars
                  > @i"fallc: Warning: variable '$(var)' in translated string ID=$(stringId) was not in the original"
               end
            end
         end

         if translated
            ltab.entries[ stringId ] = translated
         end
         
         // should we keep the originals?
         if Options.merge
            ltab.originals[ stringId ] = original
         end

         child = child.nextSibling()
      end

      > @i"fallc: Parsed file \"$(file)\""
      return ltab

   catch MXMLError in error
      > @i"fallc: Warning: Couldn't read or parse the table \"$(file)\": "
      > error
      return nil
   end
end

//====================================================
// Saving a merged language table
//

function saveMerge( lmerge, ltab, output )
   try
      outstream = OutputStream( output )
      outstream.setEncoding( "utf-8" )
   catch IoError in error
      > @i"fallc: Warning: Couldn't create output file \"$(output)\": "
      > error
      return false
   end

   // put the translations done in ltab inside lmerge,
   // but first organize a reverese dictinary of lmerge.
   reverseMap = [=>]
   for key, value in lmerge.originals
      reverseMap[value] = key
   end

   transMap = [=>]
   for key, value in ltab.originals
      try
         id = reverseMap[ value ]
         transMap[id] = ltab.entries[key]
      catch AccessError
      end
   end

   // now, write the output file
   doc = MXMLDocument()

   // add our items below root.
   root = doc.root()
   root.name( "translation" )
   root.setAttribute( "from", ltab.fromLang )
   root.setAttribute( "into", ltab.name )
   root.setAttribute( "module", ltab.modName )

   for key, value in lmerge.originals
      node_string = MXMLNode( MXMLType.tag, "string" )
      node_string.setAttribute( "id", key.toString() )
      node_o = MXMLNode( MXMLType.tag, "original" )
      node_o.data( value )
      node_t = MXMLNode( MXMLType.tag, "translated" )
      if key in transMap
         node_t.data( transMap[key] )
      end

      node_string.addBelow( node_o )
      node_string.addBelow( node_t )

      root.addBelow( node_string )
   end

   doc.style( MXMLStyle.INDENT )
   doc.setEncoding( "utf-8" )
   try
      doc.serialize( outstream )
   catch IoError in e
      > @i"fallc: Warning: Couldn't create output file \"$(output)\": "
      > error
      return false
   end

   return true
end

//====================================================
// save the tables
//
// TABLE FORMAT:
// 0-4: MARKER TLTAB
// 5-6: FBFC endianity marker
// 7-10: Number of entries (N).
// 11- (11 + (N*9)): Language index table (lexically ordered):
//   0-4: 5 bytes language code.
//   5-8: language table + table offset.
// N language tables:
//   0-3: Count of entries in the table (K)
//   K entries:
//      0-3: Id of the string in the table
//      4-7: Length of the string to be read in bytes (L)
//      8-8+L: String in UTF-8 format
//

function saveTables( languages, output )
   > @i"fallc: Saving table into $(output)"

   // we need to know all the translations.
   tables = [=>]
   for name, lang in languages
      if transcodeTo( name, "utf-8" ).len() != 5
         > @i"fallc: Warning; invalid language code $(lang.name). Ignoring."
         continue
      end

      langsize = lang.entries.len()
      > @"Saving $langsize entries for language \"$name\""
      tempStream = StringStream()
      bytes = "\xFFFFFFFF"
      bytes[0] = langsize
      tempStream.write( bytes )

      for id, entry in lang.entries
         bytes[0] = id
         tempStream.write( bytes )
         utf8entry = transcodeTo( entry, "utf-8" )
         bytes[0] = utf8entry.len()
         tempStream.write( bytes )
         tempStream.write( utf8entry )
      end

      tables[name] = tempStream.closeToString()
   end

   stream = OutputStream( output )
   stream.write( "TLTAB" )

   // This will allow the decoding program to understand what we're saying.
   endianMarker = "\xFBFC"
   stream.write( endianMarker )
   // write the count of tables
   bytes[0] = tables.len()
   stream.write( bytes )

   // then the table index
   // using the iterator avoids useless copies of big data chunks
   iter = tables.first()
   curpos = 0
   while iter.hasCurrent()
      stream.write( iter.key() )
      bytes[0] = curpos
      stream.write( bytes )
      curpos += iter.value().len()
      iter.next()
   end

   // now write the tables.
   iter = tables.first()
   while iter.hasCurrent()
      stream.write( iter.value() )
      iter.next()
   end

   stream.close()
   return true
end

//====================================================
// main code
//

Options.parse()

// nothing to do ?
if not Options.inputs and not Options.globs
   > i"fallc: Nothing to do."
   return 0
end

// expand globs
for glob in Options.globs
   // get the path part
   path = Path( glob )
   location = path.location
   filename = path.filename
   // try to diropen the location
   try
      direnum = Directory( location )
      while (file = direnum.read() )
         if strWildcardMatch( glob, file )
            Options.inputs += location + "/" + file
         end
      end
   catch IoError
      > @i"fallc: FATAL: can't read directory \"$(location)\"."
      return 1
   end
end

// parse the language table
languages = [=>]
modname = nil
for file in Options.inputs
   ltab = parseLanguage( file )

   // Continue in case of errors
   if not ltab: continue

   if modname == nil
      modname = ltab.modName
   elif modname != ltab.modName
      > @i"fallc: Warning: ignoring file \"$(ltab.file)\"."
      > @i"fallc: containing infos for module \"$(ltab.module)\"."
      continue
   end

   if ltab.name in languages
      > @i"fallc: Warning: definition for language \"$(ltab.name)\" in file \"$(ltab.file)\""
      > @i"fallc:    was already present in file \"$(languages[ltab.name])\"."
   end
   languages[ ltab.name ] = ltab
end

// try to save, if we have to.
if languages
   // time to save the table
   // use the last table as default name

   // are we in merge mode?
   if Options.merge
      Options.checkvars = false  // we don't need to check vars here.
      lmerge = parseLanguage( Options.merge )
      if lmerge
         for name, lang in languages
            output = Options.output ? Options.output : \
               modname + "." + lang.name+ ".ftt"
            saveMerge( lmerge, lang, output )
            if Options.output: break
         end
      end
   else
      output = Options.output ? Options.output : modname + ".ftr"
      try
         if saveTables( languages, output )
            > @i"fallc: Saved translation for module \"$(modname)\" to \"$(output)\""
            return 0 // OK
         end
      catch IoError
         > @i"fallc: FATAL: error while saving to \"$(output)\""
      end
   end
else
   > @i"fallc: No language availabe."
end

return 1

/* end of fallc.fal */