UTF-16 to UTF-7 File Conversion

[Frank P. Westlake]

A follow-up message contains the script "UTF16to7.cmd" which might be useful to some people for encoding Unicode e-mail and Unicode other files. It is also at:

<http://geocities.com/fp.westlake/utf16to7.zip>

This script will read a unicode (UTF-16LE or UTF-16[LE]) and output a mixture of ASCII and UTF-7.

Here are a couple of useful scripts for anyone viewng UTF-7 or UTF-8 in the console:

:: BEGIN FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::
:: Type7.cmd
:: TYPE UTF-7 to the console.
@Echo OFF
For /F "tokens=2 delims=:" %%a in ('CHCP') Do (
CHCP 65000 >NUL:
CMD /C"TYPE %*"
CHCP %%a >NUL:
)
:: END FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::::

:: BEGIN FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::
:: Type8.cmd
:: TYPE UTF-8 to the console.
@Echo OFF
For /F "tokens=2 delims=:" %%a in ('CHCP') Do (
CHCP 65001 >NUL:
CMD /C"TYPE %*"
CHCP %%a >NUL:
)
:: END FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::::

Frank

[Frank P. Westlake]

:: BEGIN FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::
:: UTF16to7.cmd
:: Output UTF-7 from a UTF-16 file.
:: Frank P. Westlake, 2009-08-01
::
@Echo OFF
SetLocal ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION
If /I "%1" EQU "/?" (
Echo.Prints UTF-7 from a UTF-16, UTF-16LE, or UTF-16BE file.
Echo.
Echo. %0 [/CI] Filename
Echo.
Echo. /CI Insert a UTF-7 content indicator at start of file.
Echo. If the UTF-16 file begins with the byte-order mark
Echo. FEFFh (FFh FEh^) those bytes are converted to the
Echo. UTF-7 content indicator so /CI should not be used.
Echo.
Echo.The output should be redirected to a file. The input file is
Echo.not harmed.
Echo.
Echo.Example:
Echo. %0 unicode.txt ^> utf-7.txt
Goto :EOF
)
Set "Me=%~n0"
:: Alterable environment:
Set "CI="
Set "MyDir=%temp%\ASCII"
Set "TmpFile=%TEMP%\%~n0"
:: End alterable environment
If "%1" EQU ":WriteBinaryFiles" (Shift & Goto :WriteBinaryFiles)
:args
If /I "%1" EQU "/CI" (
Set "CI=1"
Shift
) Else If DEFINED File (
Echo.%Me%: Too many filenames. >&2
Goto :EOF
) Else (
Set "FileIn=%~f1"
Shift
)
IF "%1" NEQ "" Goto :args
If "%FileIn%" EQU "" (
Set /P "FileIn=%Me%: Please enter the name of the UTF-16LE file: " >&2
)
If "%FileIn%" EQU "" (Echo.%Me%: Aborting. Need filename. >&2 & Goto :EOF)
Set "HX=0123456789ABCDEF"
Set "s64=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
Set "ans="
Set "now="
Start "" /wait /MIN %Me% :WriteBinaryFiles %MyDir%
ChDir /d %MyDir%
For %%f in (%FileIn%) Do Set fs=%%~zf
Set "FSUtil=1"
If NOT EXIST FSUTIL.EXE (
For %%f in (FSUTIL.EXE) Do (
If NOT EXIST %%~$PATH:f (
Set "FSUtil="
)
)
)
If DEFINED FSUTIL (
FSUtil FILE CREATENEW %TmpFile%.fc %fs% >NUL:
) Else (
TYPE NUL: >%TmpFile%.fc
For /L %%i in (1 1 %fs%) Do TYPE ASCII00.0 >>%TmpFile%.fc
)
Set /a idx=-1, cnt=0, tot=1, odd=0
:::::::::::::::::::::::::::::::::::::::::::::::::::::::
Type NUL: >%TmpFile%.w
For /F "skip=1 tokens=1,2 delims=: " %%a in (
'FC /b %FileIn% %TmpFile%.fc') Do (
Set /A "tot+=1, lst=idx, idx=0x%%a, dif=idx-lst"
For /L %%L in (!dif! -1 2) Do (
Set /A tot+=1
Echo 00 >>%TmpFile%.w
)
Echo %%b >>%TmpFile%.w
)
For /L %%i in (%tot% 1 %fs%) Do Echo 00 >>%TmpFile%.w
If DEFINED CI (
For %%b in (2B 2F 76 38 2D) Do Type ASCII%%b.* 2>NUL:
)
For /F %%b in (%TmpFile%.w) Do (
Set /A "rcv=0x%%b, odd=~odd, odd&=1"
If !odd! EQU 1 (
Set /A "val=!rcv!"
) Else (
Set /A "val|=!rcv!<<8, cnt+=1"
REM CR LF TAB
If !val! EQU 0x0D (Set "now=1"
) Else If !val! EQU 0x0A (Set "now=1"
) Else If !val! EQU 0x09 (Set "now=1"
) Else If !val! GTR 0x1F (REM SPACE to ~ except + and \.
If !val! LSS 0x7E (REM ~
If !val! NEQ 0x2B (REM +
IF !val! NEQ 0x5C (REM \
Set "now=1"
)
)
)
)
)
If NOT DEFINED now (
If !odd! EQU 0 (
Set "sto=!sto! !val!"
Set /A val=0
)
)
If DEFINED now (
If DEFINED sto (Call :UtoUTF7 !sto! & Set "sto=")
If DEFINED ans (Set /P "=+!ans!-" <NUL: & Set "ans=")
Type ASCII??.!val! 2>NUL:
Set /A cnt=0, val=0 & Set "now="
) Else (
If !cnt! EQU 3 (
If DEFINED sto (Call :UtoUTF7 !sto! & Set "sto=")
Set /A cnt=0, val=0
)
)
)
:::::::::::::::::::::::::::::::::::::::::::::::::::::::
If DEFINED ans Set /P "=+!ans!-" <NUL:
For %%x in (fc w) Do Erase %TmpFile%.%%x
Goto :EOF

:UtoUTF7
Set "rep="
Set "cmp=%1 %2 %3"
Set i1=%1&Shift
Set /A "c1=i1>>10, c2=(i1>>4)&0x3F, c3=(i1&0xF)<<2"
Set "rep=%rep%!!s64:~%c1%,1!!!!s64:~%c2%,1!!"
If "%1" EQU "" (Set "rep=%rep%!!s64:~%c3%,1!!" & Goto :UtoUTF7.set)

Set i1=%1&Shift
Set /A "c3|=(i1>>14), c4=(i1>>8)&0x3F, c5=(i1>>2)&0x3F, c6=(i1&0x3)<<4"
Set "rep=%rep%!!s64:~%c3%,1!!!!s64:~%c4%,1!!!!s64:~%c5%,1!!"
If "%1" EQU "" (Set "rep=%rep%!!s64:~%c6%,1!!" & Goto :UtoUTF7.set)

Set i1=%1&Shift
Set /A "c6|=(i1>>12)&0xF, c7=(i1>>6)&0x3F, c8=i1&0x3F"
Set "rep=%rep%!!s64:~%c6%,1!!!!s64:~%c7%,1!!!!s64:~%c8%,1!!"

:UtoUTF7.set
If "%rep%" EQU "AAA" Goto :EOF
If "%rep%" EQU "AAAAAA" Goto :EOF
If "%rep%" EQU "AAAAAAAA" Goto :EOF
Set "ans=%ans%%rep%"
Goto :EOF

:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
:WriteBinaryFiles path
SetLocal
MkDir %MyDir% >NUL: 2>&1
ChDir /d %MyDir%
FOR /L %%i in (0 1 0x7F) Do If NOT EXIST ASCII*.%%i (
Set /A "h1=(%%i&0xF0)>>4, h2=(%%i&0x0F)"
Call Set "h=%%HX:~!h1!,1%%%%HX:~!h2!,1%%"
(
CALL Echo N ascii%%h%%.%%i
CALL Echo E 0000 %%h%%
Echo R CX
Echo 1
Echo W 0
Echo Q
) | DEBUG >NUL:
)
Exit
Goto :EOF
:: END FILE ::::::::::::::::::::::::::::::::::::::::::::::::::::::