New ML support - optimized script lang-build.sh

lang/lang-build.sh

@@ -4,39 +4,35 @@
 #  generate lang_xx.bin (language binary file)
 #
 # Input files:
-#  lang_en.txt
-#  lang_en_xx.txt
-#
+#  lang_en.txt or lang_en_xx.txt
+#  
 # Output files:
+#  lang_xx.bin
 #
-#
-# Selected language:
-LNG=$1
-if [ -z "$LNG" ]; then LNG='cz'; fi
+# Temporary files:
+#  lang_xx.tmp
+#  lang_xx.dat
 #
 
 #awk code to format ui16 variables for dd
 awk_ui16='{ h=int($1/256); printf("\\x%02x\\x%02x\n", int($1-256*h), h); }'
 
+#startup message
+echo "lang-build.sh started" >&2
+
 #exiting function
 finish()
 {
  if [ $1 -eq 0 ]; then
-  if [ -e lang_en.tmp ]; then rm lang_en.tmp; fi
-  if [ -e lang_en_$LNG.tmp ]; then rm lang_en_$LNG.tmp; fi
-  if [ -e lang_en_$LNG.dif ]; then rm lang_en_$LNG.dif; fi
- fi
-# echo >&2
- if [ $1 -eq 0 ]; then
-  echo "make_lang.sh finished with success" >&2
+  echo "lang-build.sh finished with success" >&2
  else
-  echo "make_lang.sh finished with errors!" >&2
+  echo "lang-build.sh finished with errors!" >&2
  fi
  exit $1
 }
 
 #returns hexadecial data for lang code
-lang_code()
+lang_code_hex_data()
 # $1 - language code ('en', 'cz'...)
 {
  case "$1" in
@@ -51,7 +47,6 @@ lang_code()
  echo '??'
 }
 
-#
 write_header()
 # $1 - lang
 # $2 - size
@@ -67,7 +62,7 @@ write_header()
   dd of=lang_$1.bin bs=1 count=2 seek=6 conv=notrunc 2>/dev/null
  /bin/echo -n -e $(echo -n "$(($4))" | awk "$awk_ui16") |\
   dd of=lang_$1.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null
- /bin/echo -n -e "$(lang_code $1)" |\
+ /bin/echo -n -e "$(lang_code_hex_data $1)" |\
   dd of=lang_$1.bin bs=1 count=2 seek=10 conv=notrunc 2>/dev/null
  sig_h=$(($5 / 65536))
  /bin/echo -n -e $(echo -n "$sig_h" | awk "$awk_ui16") |\
@@ -77,176 +72,67 @@ write_header()
   dd of=lang_$1.bin bs=1 count=2 seek=12 conv=notrunc 2>/dev/null
 }
 
-make_lang2()
-# $1 - lang ('en', 'cz'...)
+generate_binary()
+# $1 - language code ('en', 'cz'...)
 {
- rm -f lang_$LNG.tmp
- rm -f lang_$LNG.dat
- rm -f lang_$LNG.bin
+ echo "lang="$1 >&2
+ #remove output and temporary files
+ rm -f lang_$1.bin
+ rm -f lang_$1.tmp
+ rm -f lang_$1.dat
  LNG=$1
  #create lang_xx.tmp - different processing for 'en' language
- if [ "$LNG" = "en" ]; then
+ if [ "$1" = "en" ]; then
   #remove comments and empty lines
   cat lang_en.txt | sed '/^$/d;/^#/d'
  else
   #remove comments and empty lines, print lines with translated text only
-  cat lang_en_$LNG.txt | sed '/^$/d;/^#/d' | sed -n 'n;p'
- fi | sed 's/^\"\\x00\"$/\"\"/' > lang_$LNG.tmp
+  cat lang_en_$1.txt | sed '/^$/d;/^#/d' | sed -n 'n;p'
+ fi | sed 's/^\"\\x00\"$/\"\"/' > lang_$1.tmp
  #create lang_xx.dat (binary text data file)
- cat lang_$LNG.tmp | sed 's/^\"/printf \"/;s/"$/\\x00\"/' | sh >lang_$LNG.dat
+ cat lang_$1.tmp | sed 's/^\"/printf \"/;s/"$/\\x00\"/' | sh >lang_$1.dat
  #calculate number of strings
- count=$(grep -c '^"' lang_$LNG.tmp)
- echo "count="$count
+ count=$(grep -c '^"' lang_$1.tmp)
+ echo "count="$count >&2
  #calculate text data offset
  offs=$((16 + 2 * $count))
- echo "offs="$offs
+ echo "offs="$offs >&2
  #calculate text data size
- size=$(($offs + $(wc -c lang_$LNG.dat | cut -f1 -d' ')))
- echo "size="$size
+ size=$(($offs + $(wc -c lang_$1.dat | cut -f1 -d' ')))
+ echo "size="$size >&2
  #write header with empty signature and checksum
- write_header $LNG $size $count 0x0000 0x00000000
+ write_header $1 $size $count 0x0000 0x00000000
  #write offset table
- cat lang_$LNG.tmp | sed 's/^\"//;s/\"$//' |\
+ offs_hex=$(cat lang_$1.tmp | sed 's/^\"//;s/\"$//' |\
   sed 's/\\x[0-9a-f][0-9a-f]/\./g;s/\\[0-7][0-7][0-7]/\./g;s/\ /\./g' |\
-  awk 'BEGIN { o='$offs';} { h=int(o/256); printf("%c%c",int(o-256*h), h); o+=(length($0)+1); }' |\
-  dd of=./lang_$LNG.bin bs=1 seek=16 conv=notrunc 2>/dev/null
+  awk 'BEGIN { o='$offs';} { h=int(o/256); printf("\\x%02x\\x%02x",int(o-256*h), h); o+=(length($0)+1); }')
+ /bin/echo -n -e "$offs_hex" | dd of=./lang_$1.bin bs=1 seek=16 conv=notrunc 2>/dev/null
  #write binary text data
- dd if=./lang_$LNG.dat of=./lang_$LNG.bin bs=1 seek=$offs conv=notrunc 2>/dev/null
+ dd if=./lang_$1.dat of=./lang_$1.bin bs=1 seek=$offs conv=notrunc 2>/dev/null
+ #write signature
+ if [ "$1" != "en" ]; then
+  dd if=lang_en.bin of=lang_$1.bin bs=1 count=4 skip=6 seek=12 conv=notrunc 2>/dev/null
+ fi
  #calculate and update checksum
- chsum=$(cat lang_$LNG.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }')
+ chsum=$(cat lang_$1.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }')
  /bin/echo -n -e $(echo -n $((0x$chsum)) | awk "$awk_ui16") |\
-  dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null
+  dd of=lang_$1.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null
  #remove temporary files
- rm -f lang_$LNG.tmp
- rm -f lang_$LNG.dat
-}
-
-make_lang2 $1
-exit
-
-make_lang()
-{
-LNG=$1
-
-echo "make_lang.sh started" >&2
-echo "selected language=$LNG" >&2
-
-#check if input files exists
-echo -n " checking input files..." >&2
-if [ ! -e lang_en.txt ]; then echo "NG!  file lang_en.txt not found!" >&2; exit 1; fi
-
-if [ ! -e lang_en_$LNG.txt ]; then echo "NG!  file lang_en_$LNG.txt not found!" >&2; exit 1; fi
-echo "OK" >&2
-
-#filter comment and empty lines from key and dictionary files, create temporary files
-echo -n " creating tmp files..." >&2
-cat lang_en.txt | sed "/^$/d;/^#/d" > lang_en.tmp
-cat lang_en_$LNG.txt | sed "/^$/d;/^#/d" > lang_en_$LNG.tmp
-echo "OK" >&2
-#cat lang_en_$LNG.tmp | sed 'n;d' >test1.txt
-
-#compare files using diff and check for differences
-echo -n " comparing tmp files..." >&2
-if ! cat lang_en_$LNG.tmp | sed 'n;d' | diff lang_en.tmp - > lang_en_$LNG.dif; then
- echo "NG!" >&2
- echo "Entries in lang_en_$LNG.txt are different from lang_en.txt!" >&2
- echo "please check lang_en_$LNG.dif" >&2
- finish 1
-fi
-echo "OK" >&2
-
-#generate lang_xx.txt (secondary language text data sorted by ids)
-echo -n " generating lang_$LNG.txt..." >&2
-cat lang_en_$LNG.tmp | sed '1~2d' | sed "s/^\"\\\\x00/\"/" > lang_$LNG.txt
-echo "OK" >&2
-
-#generate lang_xx.dat (secondary language text data in binary form)
-echo -n " generating lang_$LNG.dat..." >&2
-cat lang_$LNG.txt | sed "s/\\\\/\\\\\\\\/g" | while read s; do
- s=${s#\"}
- s=${s%\"}
- /bin/echo -e -n "$s\x00"
-done >lang_$LNG.dat
-echo "OK" >&2
-
-#calculate variables
-lt_magic='\xa5\x5a\xb4\x4b'
-lt_count=$(grep -c '^' lang_$LNG.txt)
-lt_data_size=$(wc -c lang_$LNG.dat | cut -f1 -d' ')
-lt_offs_size=$((2 * $lt_count))
-lt_size=$((16 + $lt_offs_size + $lt_data_size))
-lt_chsum=0
-lt_code='\xff\xff'
-lt_resv1='\xff\xff\xff\xff'
-
-case "$LNG" in
- *en*) lt_code='\x6e\x65' ;;
- *cz*) lt_code='\x73\x63' ;;
- *de*) lt_code='\x65\x64' ;;
- *es*) lt_code='\x73\x65' ;;
- *it*) lt_code='\x74\x69' ;;
- *pl*) lt_code='\x6c\x70' ;;
-esac
-
-#generate lang_xx.ofs (secondary language text data offset table)
-echo -n " generating lang_$LNG.ofs..." >&2
-cat lang_$LNG.txt | sed "s/\\\\x[0-9a-f][0-9a-f]/\./g;s/\\\\[0-7][0-7][0-7]/\./g" |\
- awk 'BEGIN { o='$((16 + $lt_offs_size))';} { printf("%d\n",o); o+=(length($0)-1); }' > lang_$LNG.ofs
-echo "OK" >&2
-
-#generate lang_xx.bin (secondary language result binary file)
-echo " generating lang_$LNG.bin:" >&2
-#create empty file
-dd if=/dev/zero of=lang_$LNG.bin bs=1 count=$lt_size 2>/dev/null
-
-#write data to binary file with dd
-
-echo -n "  writing header (16 bytes)..." >&2
-/bin/echo -n -e "$lt_magic" |\
- dd of=lang_$LNG.bin bs=1 count=4 seek=0 conv=notrunc 2>/dev/null
-/bin/echo -n -e $(echo -n "$lt_size" | awk "$awk_ui16") |\
- dd of=lang_$LNG.bin bs=1 count=2 seek=4 conv=notrunc 2>/dev/null
-/bin/echo -n -e $(echo -n "$lt_count" | awk "$awk_ui16") |\
- dd of=lang_$LNG.bin bs=1 count=2 seek=6 conv=notrunc 2>/dev/null
-/bin/echo -n -e $(echo -n "$lt_chsum" | awk "$awk_ui16") |\
- dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null
-/bin/echo -n -e "$lt_code" |\
- dd of=lang_$LNG.bin bs=1 count=2 seek=10 conv=notrunc 2>/dev/null
-/bin/echo -n -e "$lt_resv1" |\
- dd of=lang_$LNG.bin bs=1 count=4 seek=12 conv=notrunc 2>/dev/null
-echo "OK" >&2
-
-echo -n "  writing offset table ($lt_offs_size bytes)..." >&2
-/bin/echo -n -e $(cat lang_$LNG.ofs | awk "$awk_ui16" | tr -d '\n'; echo) |\
- dd of=./lang_$LNG.bin bs=1 count=$lt_offs_size seek=16 conv=notrunc 2>/dev/null
-echo "OK" >&2
-
-echo -n "  writing text data ($lt_data_size bytes)..." >&2
-dd if=./lang_$LNG.dat of=./lang_$LNG.bin bs=1 count=$lt_data_size seek=$((16 + $lt_offs_size)) conv=notrunc 2>/dev/null
-echo "OK" >&2
-
-#calculate and update checksum
-lt_chsum=$(cat lang_$LNG.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }')
-/bin/echo -n -e $(echo -n $((0x$lt_chsum)) | awk "$awk_ui16") |\
- dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null
-
-echo " lang_table details:" >&2
-echo "  lt_count = $lt_count" >&2
-echo "  lt_size  = $lt_size" >&2
-echo "  lt_chsum = $lt_chsum" >&2
+ rm -f lang_$1.tmp
+ rm -f lang_$1.dat
 }
 
-echo $LNG
+if [ -z "$1" ]; then set 'all'; fi
 
-if [ "$LNG" = "all" ]; then
- make_lang cz
- make_lang de
- make_lang es
- make_lang it
- make_lang pl
- exit 0
+if [ "$1" = "all" ]; then
+ generate_binary 'en'
+ generate_binary 'cz'
+ generate_binary 'de'
+ generate_binary 'es'
+ generate_binary 'it'
+ generate_binary 'pl'
 else
- make_lang $LNG
+ generate_binary $1
 fi
 
 finish 0