diff --git a/CREDITS b/CREDITS new file mode 100644 index 0000000..e94f8a9 --- /dev/null +++ b/CREDITS @@ -0,0 +1,10 @@ +I'd like to thank the following people for ideas, patches, testing or other +useful stuff: + +Mikael MAGNUSSON +Mike FABIAN +Philipp JOCHAM +Kuang-che Wu +Hansjörg MAURER +Gabor PAPP +Bruno HAIBLE diff --git a/Changes b/Changes new file mode 100644 index 0000000..dfe3f2e --- /dev/null +++ b/Changes @@ -0,0 +1,97 @@ +2.01 +- map : instead of / in SFU mapping tables + +2.0 +- fix checks for NFD conversion, where convmv could run into a "resulting filename is ... bytes long (max: 255) error message for no obvious reason. +- the --preserve-mtimes option is the default now +- fix a bug where mtimes might not be restored in some cases +- add --map option to support additional character mappings like to mapping Microsoft's illegal NTFS characters +- issue warning if we cannot traverse a directory in recursive mode + +1.15 +- add --dump-options to make it easier to allow people to write a bash completion file. Let me know when you have one ready :-) +- fix a bug where on case-insensitive filesystems --upper and --lower did not work because we thought the new file would already exist (stat "foo" and stat "FOO" are the same, yeah. The obvious workaround to use "--replace" would delete the files on case-insensitive filesystems instead, which was even worse. Case conversion should be working well now, I tested it on Darwin and on case-insensitive XFS. The latter one is a bit buggy by the way, see https://bugzilla.kernel.org/show_bug.cgi?id=39512. convmv works around that bug, too. + +1.14 +- this release is once more dedicated to Perl::Encode. There is no way to do a check for valid UTF-8 which works with all Perl releases from 5.8.0 up to 5.10.0. We now have to test which code works in our Perl's Encode version and then decide which code path to go. + +1.13 +- move non fatal Perl bug checks into the help screen output +- fix Perl warnings when wrong/unknown charset is used +- run check for some essential Encode functionality at every startup +- add --fixdouble option to only fix files that are double-UTF-8 encoded +- add --parsable option to allow convmv output parsable suggestions for external tools to to something with it +- add --preserve-mtimes option to restore mtimes of parent directories +- allow an argv to be a symlink +- don't checksum deleted files in tar ball and use sha256 now +- allow skippig of certain test suite tests, useful for OS X to skip NFC/NFD related stuff, for example "make test SKIP=nfd-test,someothertest" will skip that test(s) + +1.12 +- now work with Perl 5.10 using an eval block +- GPL v2 and v3 now + +1.11 +- test suite compares sorted results of find to fix false alarms on some systems +- test suite now uses tar instead of cp for copying. +- fix a y/n mixup +- at startup we check if Perl::Encode is not too broken (bugs 37757 and 49830 for now). Recent Perl releases are terribly broken. If this won't improve, I'll have to rewrite convmv in a language with more stable core functionality :-|. If you are distributor of convmv, please run make test before packaging to make sure you don't ship convmv with a broken Perl release. + +1.10 +- now use from_to instead of utf8::decode (which is experimental and might vanish in the future) to check for valid UTF-8 +- updates and cleanups of testsuite and include "0" filename +- fix "0" filename bugs +- take into account that charset name "utf-8" resolves to "utf-8-strict" in recent Perl versions + +1.09 +- work around broken decode_utf8() in Perl 5.8.7 resp. Encode 2.10 (use utf8::decode instead) +- adopt test suite to catch cases like this +- warn about dry test run at start, not just at end of a test run + +1.08 +- do not complain about wrong from_enc if to_enc is utf8 and files are, too; this happens with from_enc being a multibyte encoding. Thanks to Kuang-che Wu +- fixed bug, where filename "0" was considered to be wrong utf-8 if from_enc was utf8 +- some documentations updates + +1.07 +- add%20option%20to%20undo%20ugly%20http-%20and%20ftp-like%20escape%20sequences +- add option to upper/lowercase virtually any charset +- for upper/lower conversion add a "dotless i" option for Lithuanian, Turkish and Azeri +- check for characters not allowed in POSIX filesystems (just in case someone wants to try to convert to UTF-16) + +1.06 +- fix some typos and cosmetics +- add missing from_print +- rewrote checkenc(), no more false alarms now with "encodings" like MIME-Q, which can look quite different. As a side effect error messages became more detailed + +1.05 +- --qfrom/to also mask cntrl characters now +- symlink printout splitted to get correct output +- missing $arg in symlink fixed (bug introduced in 1.02) +- add NFS4 issues to documentation +- add test suite +- add file length check + +1.04 +- add --exec option to allow arbitrary operations + +1.03 +- add --lowmem option to keep memory footprint low when millions of files are being converted. However this will deactivate symlink target checks. +- add Samba issues to documentation +- suppress normalization error messages in cases of invalid UTF-8 in checkenc() + +1.02 +- make default behaviour to not work recursively so we can convert a single directory, use -r to get the old (recursive) mode +- work around find() oddity, which doesn't process directories from @ARGV +- die on unknown option +- check if files exist and die otherwise + +1.01 +- documentation updates +- check if symlink target is -f or -d, we run into problems in /proc otherwise. +- implement interactive (-i) mode + +1.0 +- UTF-8 smartness added + +0.3 +- first stable release diff --git a/GPL2 b/GPL2 new file mode 100644 index 0000000..86568a9 --- /dev/null +++ b/GPL2 @@ -0,0 +1,341 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 675 Mass Ave, Cambridge, MA 02139, USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) 19yy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) 19yy name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fdaf8bf --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +DESTDIR= +PREFIX=/usr/local +MANDIR=$(PREFIX)/share/man +TAR=tar +FIND=find +SED=sed + +all: manpage + +install: all + mkdir -p $(DESTDIR)$(MANDIR)/man1/ + mkdir -p $(DESTDIR)$(PREFIX)/bin/ + cp convmv.1.gz $(DESTDIR)$(MANDIR)/man1/ + install -m 755 convmv $(DESTDIR)$(PREFIX)/bin/ + +manpage: + pod2man --section 1 --center=" " convmv | gzip > convmv.1.gz + +clean: + rm -f convmv.1.gz convmv-*.tar.gz MD5sums SHA256sums .files .name + rm -rf suite + +test: + test -d suite || $(TAR) xf testsuite.tar + cd suite ; ./dotests.sh + +dist: clean + $(SED) -n "2,2p" convmv |$(SED) "s/.*convmv \([^ ]*\).*/\1/" > VERSION + $(FIND) . -name "*" ! -name ".*" -type f -print | xargs sha256sum | gpg --clearsign > .SHA256sums + mv .SHA256sums SHA256sums + ls > .files + echo convmv-`cat VERSION` >.name + mkdir `cat .name` + mv `cat .files` `cat .name` + $(TAR) cvf - * |gzip > `cat .name`.tar.gz + mv `cat .name`/* . + rmdir `cat .name` diff --git a/SHA256sums b/SHA256sums new file mode 100644 index 0000000..453743d --- /dev/null +++ b/SHA256sums @@ -0,0 +1,18 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +6cd57b452e9e2be5a9ca992831f96e4a8317c1bc37171f237094d8dc0a6e0d27 ./TODO +f759d3169c063594b04a9b1714a52e09f881dd7bc49ff65cf1c468d4e6a8dbbc ./convmv +31a04f0b0584237d6f81fab7e77b0d43e307230102f09db947e20b2a3103a108 ./testsuite.tar +42b2cfe802e3bac5bdc2462a0d61de002991125053e41c18d5cd7799b4cebf2b ./CREDITS +4d7d6c70f3f7bd5127794e01ccfc42b1f3dcd882d229ad6c9d274104eb6db6b5 ./VERSION +9c1d31097f289010a598ed3656ee9a00b56221c8d87893db005f2635250b640b ./Changes +10f96f4cf2f61333c289f603cedd4c0f5e523d2da05486d9bea670f3877d8592 ./Makefile +a2a9cfa48ba7f7453edf1c43bb8bb1aef5fcd1b87dde11420edceee2e2528db0 ./GPL2 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1 + +iEYEAREIAAYFAlkL6Z4ACgkQdoo0s+hIejmBZwCgs6PP1YWsa9ifUTosGfwEaNeS +WJIAni4rsOvom8Nxks8Io0dHDht5Se8m +=n1Km +-----END PGP SIGNATURE----- diff --git a/TODO b/TODO new file mode 100644 index 0000000..58e3594 --- /dev/null +++ b/TODO @@ -0,0 +1,14 @@ +Does this make sense? : + +- logfile option +- convert names in uni_xlate (man 8 mount, vfat ...) +- --exclude option +- verbose mode printing actual scanned files +- --null option for "find -print0 | xargs convmv --null" usage +- map illegal ntfs characters away: " / \ * ? < > | : +- map "most" strange chracters to ASCII equivalences +- keep/restore old ctime - difficult to do! No Perl module, not + even a ctime option in standard "touch" command :-| + + +any other suggestions? diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..91972c3 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.01 diff --git a/convmv b/convmv new file mode 100755 index 0000000..3746672 --- /dev/null +++ b/convmv @@ -0,0 +1,1107 @@ +#!/usr/bin/perl +# convmv 2.01 - converts filenames from one encoding to another +# Copyright © 2003-2017 Bjoern JACKE +# +# This program comes with ABSOLUTELY NO WARRANTY; it may be copied or modified +# under the terms of the GNU General Public License version 2 or 3 as +# published by the Free Software Foundation. + +# to get a man page: +# pod2man --section 1 --center=" " convmv | gzip > convmv.1.gz + + +=head1 NAME + +convmv - converts filenames from one encoding to another + +=head1 SYNOPSIS + +B [B] FILE(S) ... DIRECTORY(S) + +=head1 OPTIONS + +=over 4 + +=item B<-f ENCODING> + +specify the current encoding of the filename(s) from which should be converted + +=item B<-t ENCODING> + +specify the encoding to which the filename(s) should be converted + +=item B<-i> + +interactive mode (ask y/n for each action) + +=item B<-r> + +recursively go through directories + +=item B<--nfc> + +target files will be normalization form C for UTF-8 (Linux etc.) + +=item B<--nfd> + +target files will be normalization form D for UTF-8 (OS X etc.). + +=item B<--qfrom> , B<--qto> + +be more quiet about the "from" or "to" of a rename (if it screws up your +terminal e.g.). This will in fact do nothing else than replace any non-ASCII +character (bytewise) with ? and any control character with * on printout, this +does not affect rename operation itself. + +=item B<--exec> command + +execute the given command. You have to quote the command and #1 will be +substituted by the old, #2 by the new filename. Using this option link +targets will stay untouched. Have in mind that #1 and #2 will be quoted +by convmv already, you must not add extra quotation marks around them. + +Example: + +convmv -f latin1 -t utf-8 -r --exec "echo #1 should be renamed to #2" path/to/files + +=item B<--list> + +list all available encodings. To get support for more Chinese or Japanese +encodings install the Perl HanExtra or JIS2K Encode packages. + +=item B<--lowmem> + +keep memory footprint low by not creating a hash of all files. This disables +checking if symlink targets are in subtree. Symlink target pointers will be +converted regardlessly. If you convert multiple hundredthousands or millions of +files the memory usage of convmv might grow quite high. This option would help +you out in that case. + +=item B<--nosmart> + +by default convmv will detect if a filename is already UTF8 encoded and will +skip this file if conversion from some charset to UTF8 should be performed. +C<--nosmart> will also force conversion to UTF-8 for such files, which might +result in "double encoded UTF-8" (see section below). + +=item B<--fixdouble> + +using the C<--fixdouble> option convmv does only convert files which will still +be UTF-8 encoded after conversion. That's useful for fixing double-encoded +UTF-8 files. All files which are not UTF-8 or will not result in UTF-8 after +conversion will not be touched. Also see chapter "How to undo double UTF-8 ..." +below. + +=item B<--notest> + +Needed to actually rename the files. By default convmv will just print what it +wants to do. + +=item B<--parsable> + +This is an advanced option that people who want to write a GUI front end will +find useful (some others maybe, too). It will convmv make print out what it +would do in an easy parsable way. The first column contains the action or some +kind of information, the second column mostly contains the file that is to be +modified and if appropriate the third column contains the modified value. Each +column is separated by \0\n (nullbyte newline). Each row (one action) is +separated by \0\0\n (nullbyte nullbyte newline). + +=item B<--no-preserve-mtimes> + +modifying filenames usually causes the parent directory's mtime being updated. +Since version 2 convmv by default resets the mtime to the old value. If your +filesystem supports sub-second resolution the sub-second part of the atime and +mtime will be lost as Perl does not yet support that. With this option you can +B the preservation of the mtimes. + +=item B<--replace> + +if the file to which shall be renamed already exists, it will be overwritten if +the other file content is equal. + +=item B<--unescape> + +this option will remove this ugly % hex sequences from filenames and turn them +into (hopefully) nicer 8-bit characters. After --unescape you might want to do +a charset conversion. This sequences like %20 etc. are sometimes produced when +downloading via http or ftp. + +=item B<--upper> , B<--lower> + +turn filenames into all upper or all lower case. When the file is not +ASCII-encoded, convmv expects a charset to be entered via the -f switch. + +=item B<--map=>some-extra-mapping + +apply some custom character mappings, currently supported are: + +ntfs-sfm(-undo), ntfs-sfu(-undo) for the mapping of illegal ntfs characters for +Linux or Macintosh cifs clients (see MS KB 117258 also mapchars mount option of +mount.cifs on Linux). + +ntfs-pretty(-undo) for for the mapping of illegal ntfs characters to pretty +legal Japanese versions of them. + +See the map_get_newname() function how to easily add own mappings if needed. +Let me know if you think convmv is missing some useful mapping here. + +=item B<--dotlessi> + +care about the dotless i/I issue. A lowercase version of "I" will also be +dotless while an uppercase version of "i" will also be dotted. This is an +issue for Turkish and Azeri. + +By the way: The superscript dot of the letter i was added in the Middle Ages to +distinguish the letter (in manuscripts) from adjacent vertical strokes in such +letters as u, m, and n. J is a variant form of i which emerged at this time and +subsequently became a separate letter. + +=item B<--help> + +print a short summary of available options + +=item B<--dump-options> + +print a list of all available options + +=back + +=head1 DESCRIPTION + +B is meant to help convert a single filename, a directory tree and the +contained files or a whole filesystem into a different encoding. It just +converts the filenames, not the content of the files. A special feature of +convmv is that it also takes care of symlinks, also converts the symlink target +pointer in case the symlink target is being converted, too. + +All this comes in very handy when one wants to switch over from old 8-bit +locales to UTF-8 locales. It is also possible to convert directories to UTF-8 +which are already partly UTF-8 encoded. convmv is able to detect if certain +files are UTF-8 encoded and will skip them by default. To turn this smartness +off use the C<--nosmart> switch. + +=head2 Filesystem issues + +Almost all POSIX filesystems do not care about how filenames are encoded, here +are some exceptions: + +=head3 HFS+ on OS X / Darwin + +Linux and (most?) other Unix-like operating systems use the so called +normalization form C (NFC) for its UTF-8 encoding by default but do not enforce +this. Darwin, the base of the Macintosh OS enforces normalization form D +(NFD), where a few characters are encoded in a different way. On OS X it's not +possible to create NFC UTF-8 filenames because this is prevented at filesystem +layer. On HFS+ filenames are internally stored in UTF-16 and when converted +back to UTF-8, for the underlying BSD system to be handable, NFD is created. +See http://developer.apple.com/qa/qa2001/qa1173.html for defails. I think it +was a very bad idea and breaks many things under OS X which expect a normal +POSIX conforming system. Anywhere else convmv is able to convert files from NFC +to NFD or vice versa which makes interoperability with such systems a lot +easier. + +=head3 JFS + +If people mount JFS partitions with iocharset=utf8, there is a similar problem, +because JFS is designed to store filenames internally in UTF-16, too; that is +because Linux' JFS is really JFS2, which was a rewrite of JFS for OS/2. JFS +partitions should always be mounted with iocharset=iso8859-1, which is also the +default with recent 2.6.6 kernels. If this is not done, JFS does not behave +like a POSIX filesystem and it might happen that certain files cannot be +created at all, for example filenames in ISO-8859-1 encoding. Only when +interoperation with OS/2 is needed iocharset should be set according to your +used locale charmap. + +=head3 NFS4 + +Despite other POSIX filesystems RFC3530 (NFS 4) mandates UTF-8 but also says: +"The nfs4_cs_prep profile does not specify a normalization form. A later +revision of this specification may specify a particular normalization form." In +other words, if you want to use NFS4 you might find the conversion and +normalization features of convmv quite useful. + +=head3 FAT/VFAT and NTFS + +NTFS and VFAT (for long filenames) use UTF-16 internally to store filenames. +You should not need to convert filenames if you mount one of those filesystems. +Use appropriate mount options instead! + +=head2 How to undo double UTF-8 (or other) encoded filenames + +Sometimes it might happen that you "double-encoded" certain filenames, for +example the file names already were UTF-8 encoded and you accidently did +another conversion from some charset to UTF-8. You can simply undo that by +converting that the other way round. The from-charset has to be UTF-8 and the +to-charset has to be the from-charset you previously accidently used. If you +use the C<--fixdouble> option convmv will make sure that only files will be +processed that will still be UTF-8 encoded after conversion and it will leave +non-UTF-8 files untouched. You should check to get the correct results by doing +the conversion without C<--notest> before, also the C<--qfrom> option might be +helpful, because the double utf-8 file names might screw up your terminal if +they are being printed - they often contain control sequences which do funny +things with your terminal window. If you are not sure about the charset which +was accidently converted from, using C<--qfrom> is a good way to fiddle out the +required encoding without destroying the file names finally. + +=head2 How to repair Samba files + +When in the smb.conf (of Samba 2.x) there hasn't been set a correct "character +set" variable, files which are created from Win* clients are being created in +the client's codepage, e.g. cp850 for western european languages. As a result +of that the files which contain non-ASCII characters are screwed up if you "ls" +them on the Unix server. If you change the "character set" variable afterwards +to iso8859-1, newly created files are okay, but the old files are still screwed +up in the Windows encoding. In this case convmv can also be used to convert the +old Samba-shared files from cp850 to iso8859-1. + +By the way: Samba 3.x finally maps to UTF-8 filenames by default, so also when +you migrate from Samba 2 to Samba 3 you might have to convert your file names. + +=head2 Netatalk interoperability issues + +When Netatalk is being switched to UTF-8 which is supported in version 2 then +it is NOT sufficient to rename the file names. There needs to be done more. See +http://netatalk.sourceforge.net/2.0/htmldocs/upgrade.html#volumes-and-filenames +and the uniconv utility of Netatalk for details. + +=head1 SEE ALSO + +L L L + +=head1 BUGS + +no bugs or fleas known + +=head1 DONATE + +You can support convmv by doing a donation, see L + +=head1 AUTHOR + +Bjoern JACKE + +Send mail to bjoern [at] j3e.de for bug reports and suggestions. + +=cut + +require 5.008; +use Getopt::Long; +use File::Find; +use File::Basename; +use Cwd; +use Encode 'from_to','encode_utf8','decode_utf8','_utf8_on','_utf8_off'; +#use Encode 'is_utf8'; +use Unicode::Normalize; +use utf8; +use bytes; + +Getopt::Long::Configure ("bundling"); +binmode STDOUT, ":bytes"; +binmode STDERR, ":bytes"; + +my $opt_mtimes = 1; # default 1 since convmv 2.0 +my %opts = ( + 'nfc'=>\$opt_nfc, + 'nfd'=>\$opt_nfd, + 'f=s'=>\$opt_f, + 't=s'=>\$opt_t, + 'r'=>\$opt_r, + 'i'=>\$opt_i, + 'list'=>\$opt_list, + 'help'=>\$opt_help, + 'notest'=>\$opt_notest, + 'qfrom'=>\$opt_qfrom, + 'qto'=>\$opt_qto, + 'replace'=>\$opt_replace, + 'nosmart'=>\$opt_nosmart, + 'lowmem'=>\$opt_lowmem, + 'exec=s'=>\$opt_exec, + 'unescape'=>\$opt_unescape, + 'upper'=>\$opt_upper, + 'lower'=>\$opt_lower, + 'dotlessi'=>\$opt_dotlessi, + 'parsable'=>\$opt_parsable, + 'fixdouble'=>\$opt_fixdouble, + 'preserve-mtimes!'=>\$opt_mtimes, + 'dump-options'=>\$opt_dumpoptions, + 'undo-script=s'=>\$opt_undo_script, + 'map=s'=>\$opt_map, + ); +GetOptions %opts or exit 1; +use File::Compare; +$errors_occurred=0; +$warnings_occurred=0; +$ops=0; +$mytime = time(); +$maxfilenamelength=255; +# $maxpathlength=4096; # this might be used somehow, somewhere? + +%dir_time_hash=(); +my $this_is_valid_utf8; + +# delimiter and final delimiter for parsable mode: +$del = "\0\n"; +$fin_del = "\0\0\n"; + +&listvalidencodings and exit 0 if ($opt_list); +&dumpoptions and exit 0 if ($opt_dumpoptions); +&printusage and exit 1 if (!@ARGV or $opt_help); + +&check_for_broken_perl_release(); + +if ($opt_parsable) { + if ($opt_notest or $opt_exec or $opt_i) { + die "--parsable mode cannot be used with --notest, --exec or -i\n"; + } +} + +if ($opt_replace and $opt_undo_script) { + die "--replace and --undo-script can't work together!\n"; +} + +if ($opt_unescape) { + die "No charset conversion when unescaping!\n" if ($opt_f or $opt_t); + $checkenc=\&unescape_checkenc; + $get_newname=\&unescape_get_newname; +} elsif ($opt_upper or $opt_lower) { + die "No charset conversion when uppering/lowering!\n" if ($opt_t); + die "Not possible to --upper and --lower at the same time!\n" if ($opt_upper and $opt_lower); + $checkenc=\&upperlower_checkenc; + $get_newname=\&upperlower_get_newname; + $opt_f="ascii" unless ($opt_f); +} elsif ($opt_map) { + if ($opt_t or $opt_f or $opt_upper or $opt_lower or $opt_unescape) { + die "--map parameter not allowed with other character conversion parameters\n"; + } + $checkenc=\&dummy; + $get_newname=\&map_get_newname; +} else { + if (not ($opt_f and $opt_f=Encode::resolve_alias($opt_f))) { + die "wrong/unknown \"from\" encoding!\n"; + } + if (not ($opt_t and $opt_t=Encode::resolve_alias($opt_t))) { + die "wrong/unknown \"to\" encoding!\n"; + } + if ($opt_fixdouble) { + $checkenc=\&fixdouble_checkenc; + } else { + $checkenc=\&char_checkenc; + } + $get_newname=\&char_get_newname; +} +$to_is_utf8 = lc($opt_t) =~ m/^utf-?8/; +$from_is_utf8 = lc($opt_f) =~ m/^utf-?8/; + +if ($opt_qfrom) { + $from_print=\&to_ascii; +} else { + $from_print=\&dummy; +} + +if ($opt_qto) { + $to_print=\&to_ascii; +} else { + $to_print=\&dummy; +} + +if ($opt_nfc) { + $norm=\&NFC; + die "NFC requires UTF-8 as target charset\n" unless ($to_is_utf8); +} elsif ($opt_nfd) { + $norm=\&NFD; + die "NFD requires UTF-8 as target charset\n" unless ($to_is_utf8); +} else { + $norm=\&dummy; +} + +if ($opt_fixdouble) { + die "--fixdouble requires UTF-8 as source and non-UTF-8 as target charset\n" unless ($from_is_utf8 and $opt_t and not $to_is_utf8); +} + +$opt_lowmem=1 if ($opt_exec); + +$pwd=cwd(); +@args=@ARGV; +undef @ARGV; + +for (@args) { + s/\/\.\//\/\//g; # normalize "/./" to "/" + s/\/[\/]+/\//g; # normalize "//" to "/" + die "file or directory not found: $_\n" unless (-e or -l); +} +if ($opt_parsable) { + $outerr=NUL; +} else { + $outerr=STDERR; +} +if ($opt_undo_script) { + die "undo-script file already exists, exiting.\n" if (-e $opt_undo_script); + open(UNDOLOG, ">", $opt_undo_script) or die "couldn't open undo-script for writing. Aborting.\n"; + print UNDOLOG "# this is a per undo script generated by convmv.\n", + "# Please check if this looks reasonable before running!\n"; + print UNDOLOG "# Example: perl $opt_undo_script\n"; + print UNDOLOG "chdir $pwd;\n"; +} + +## do {print ord($_)."_" for (split(//,$_));print "\n"; } for (@args); # debug print + +print $outerr "Starting a dry run without changes...\n" unless ($opt_notest); + +if ($opt_r) { + $myfind=\&find; +} else { + $myfind=\&find0depth; +} + +&$myfind({wanted=>\&scan,bydepth=>1,no_chdir=>1}, @args); +if (not $errors_occurred and $warnings_occurred) { + $errors_occurred=1 if (not &print_ask ("WARNINGS occurred. Do you really want to continue?",1)); +} + +die "To prevent damage to your files, we won't continue.\nFirst fix this or correct options!\n" if ($errors_occurred); +unless ($opt_exec) { + &$myfind({wanted=>\&process_symlink_targets,bydepth=>1,no_chdir=>1}, @args); +} +&$myfind({wanted=>\&process_main,bydepth=>1,no_chdir=>1}, @args); + +# check for unintentionally left files +#for (keys %dir_time_hash) { +# print $outerr "error: left in %dir_time_hash: $_\n"; +#} + +$mytime = time() - $mytime; +if ($opt_notest) { + print $outerr "Ready! I converted $ops files in $mytime seconds.\n", +} else { + print $outerr "No changes to your files done. Would have converted $ops files in $mytime seconds.\nUse --notest to finally rename the files.\n"; +} + +##### +## subs +### + +# find-like function but without any depth search for not recursive mode: +sub find0depth() { + my $opts = shift; + for (@_) { + $$opts{'wanted'}($_); + } +} + +# scan for real files and check charset first: +sub scan { + $arg = $_; + &get_dir_base_change; + if (-l $arg) { +# print "link: $arg in $dir\n"; + if (not defined(&$checkenc($arg))) { $errors_occurred=1 }; + } elsif (-d $arg) { +# print "dir: $arg in $dir\n"; + $inod_fullname{(stat $arg)[1]}=$dir."/".$arg if (!$opt_lowmem); + if (not defined(&$checkenc($arg))) { $errors_occurred=1 }; + if ($opt_r and not (-x $arg or -r $arg)) { + print $outerr "WARNING: cannot traverse ",&$from_print($dir."/".$arg),"\n"; + $warnings_occurred=1; + } + } elsif (-f $arg) { +# print "file: $arg in $dir\n"; + $inod_fullname{(stat $arg)[1]}=$dir."/".$arg if (!$opt_lowmem); + if (not defined(&$checkenc($arg))) { $errors_occurred=1 }; + } + chdir $pwd; +} + +# move symlink targets: +sub process_symlink_targets { + $arg=$_; + &get_dir_base_change; + if (-l $arg) { + $oldlink=readlink $arg; + if ((-f $oldlink or -d $oldlink) and $newname=&$get_newname($oldlink)) { + if ( $newname ne $oldlink ) { + if ( $inod_fullname{(stat $oldlink)[1]} or $opt_lowmem) { # = if (symlink target scanned before) + #print is_utf8($oldlink) ? 1 : 0; + #print is_utf8($newname) ? 1 : 0; + print $outerr "symlink \"".&$from_print($File::Find::name)."\": \""; + print $outerr "".&$from_print($oldlink)."\" >> \""; + &print_ask (&$to_print($newname)."\"",$opt_i) or return; + &save_parent_mtime($dir) if ($opt_mtimes); + if ($opt_notest) { + unlink $arg; + symlink ($newname, $arg); + print UNDOLOG "unlink \"".$File::Find::name."\";\n"; + print UNDOLOG "symlink (\"$oldlink\", \"".$File::Find::name."\");\n"; + } elsif ($opt_parsable) { + print "unlink".$del.$File::Find::name.$fin_del; + print "symlink".$del.$newname.$del.$File::Find::name.$fin_del; + } + $ops++; + } else { + print $outerr "link target \"",&$from_print($oldlink),"\" of \"",&$from_print($dir."/".$arg),"\" not in subtree, left untouched!\n"; + } + } # else { print "no need to convert link target: $oldlink to $newname\n"; } + } + } + chdir $pwd; +} + +# do the changes to all the real files/dirs/links: +sub process_main { + $arg=$_; + &get_dir_base_change; + if (-l $arg) { +# $type="symlink"; + $newname=&$get_newname($arg); + if ($newname and $newname ne $arg) { + &renameit($arg,$newname); + } + } elsif (-d $arg) { +# $type="directory"; + $newname=&$get_newname($arg); + if ($newname and $newname ne $arg) { + &renameit($arg,$newname); + } + &restore_times_if_any($dir,$arg,$newname) if ($opt_mtimes); + } elsif (-f $arg) { +# $type="file"; + $newname=&$get_newname($arg); + if ($newname and $newname ne $arg) { + &renameit($arg,$newname); + } + } + # &restore_times_if_any($dir,$arg,$newname) if ($opt_mtimes); # only in -d case needed !? -> moved up! + + chdir $pwd; +} + +sub char_get_newname { +# returns undef on error and string otherwise. + my $oldfile=shift; + my $newname; + my $lets_die = 0; + if (!$from_is_utf8 and $to_is_utf8 and !$opt_nosmart and &looks_like_utf8($oldfile)) { + if ($opt_parsable) { + print "іnfomsg".$del."skipalreadyutf8".$del.$dir."/".$oldfile.$fin_del; + } else { + print $outerr "Skipping, already UTF-8: ",&$from_print($dir."/".$oldfile),"\n"; + } + return $oldfile; + } else { + if ($opt_fixdouble and not looks_like_utf8($oldfile)) { + # this is legacy encoding which we ignore in fixdouble mode + return $oldfile; + } + if ($from_is_utf8 and ! $to_is_utf8) { + # from_to can't convert from NFD to non-UTF-8! + $newname=encode_utf8(NFC(decode_utf8($oldfile))); + } else { + $newname=$oldfile; + } + from_to($newname, $opt_f, $opt_t, Encode::FB_QUIET) or $lets_die = 1; + if ($opt_fixdouble and not looks_like_utf8($newname)) { + return $oldfile; + } + if ($lets_die) { + die "SHOULD NOT HAPPEN HERE: conversion error, no suitable charset used?: \"$oldfile\"\nTo prevent damage to your files, we won't continue. First fix this!\n"; + } + $newname=&$norm(decode_utf8($newname)) if ($to_is_utf8); + return $newname; + } + +} + +sub get_dir_base_change() { + $arg =~ s/\/*$//; + $dir=dirname($arg); + $arg=basename($arg); + chdir $dir; +} + +sub renameit() { + my $oldfile=shift; + my $newname=shift; + my $cmd; + my $ci_old_new_same_inode = 0; + $newname=encode_utf8($newname) if ($to_is_utf8); + if ($opt_exec) { + $cmd = $opt_exec; + $cmd =~ s/\#2/\000f8d9hqoäd\#2/g; # make the #2 unique so that file names may contain "#2" + $cmd =~ s/\#1/\Q$oldfile\E/g; + $cmd =~ s/\000f8d9hqoäd\#2/\Q$newname\E/g; + print "$cmd\n"; + } else { + #print is_utf8($oldfile) ? 1 : 0; + #print is_utf8($newname) ? 1 : 0; + &print_ask ("mv \"". &$from_print($dir."/".$oldfile)."\"\t\"".&$from_print($dir)."/".&$to_print($newname)."\"",$opt_i) or return; + } + &save_parent_mtime($dir) if ($opt_mtimes); + # the following is to handle case-insensitive filesystems: + if ($opt_lower or $opt_upper) { + if ((stat $oldfile)[1] == (stat $newname)[1]) { + $ci_old_new_same_inode = 1; + #print $outerr "found case-insensitive filesystem...\n"; + if ($opt_notest) { + my $try; + for (1 .. 10000) { + $try = "convmvtmp".$_; + if (! -e $try) { + #print $outerr "(via temp rename to \"$try\")\n"; + rename ($oldfile, $try); + $opt++; + print UNDOLOG "rename (\"$dir/$try\", \"$dir/$oldfile\");\n"; + $oldfile=$try; + last; + } + } + } + } + } + if (-e $newname and !$opt_exec and !$ci_old_new_same_inode) { + if ($opt_replace and !&compare($oldfile,$newname)) { + if ($opt_notest) { + unlink $newname or print $outerr "Error: $!\n"; + rename ($oldfile, $newname) or print $outerr "Error: $!\n"; + } elsif ($opt_parsable) { + print "unlink".$del.$dir."/".$oldfile.$fin_del; + print "rename".$del.$dir."/".$oldfile.$del.$dir."/".$newname.$fin_del; + } + } else { + if ($opt_parsable) { + print "errormsg".$del."fileexists".$del.$newname.$fin_del; + } else { + print $outerr "".&$to_print($newname)," exists and differs or --replace option missing - skipped\n"; + } + } + } else { + if ($opt_notest) { + if ($opt_exec) { + system($cmd); + } else { + rename ($oldfile, $newname) or print $outerr "Error: $!\n"; + print UNDOLOG "rename (\"$dir/$newname\", \"$dir/$oldfile\");\n"; + } + } elsif ($opt_parsable) { + print "rename".$del.$dir."/".$oldfile.$del.$dir."/".$newname.$fin_del; + } + } + $ops++; +} + +sub save_parent_mtime() { + my $dir=shift; + $dir =~ s/^\.\///; + $dir =~ s/\.\/$//; +# return if ($dir eq "."); # broken !? + return if (exists $dir_time_hash{$dir}); + #print $outerr "Putting \"$dir\" in %dir_time_hash\n"; # debug print + @{$dir_time_hash{$dir}}=(stat("."))[8..10]; +} + +sub restore_times_if_any() { + my $dir=shift; + my $old=shift; + my $new=shift; + if ($dir eq ".") { + $dir = ""; + } else { + $dir .= "/"; + } + $dir .= $old; + # print $outerr "Trying to delete \"$dir\" now \"$new\" from %dir_time_hash\n"; # debug print + if (exists $dir_time_hash{$dir}) { + if ($opt_notest) { + utime ${$dir_time_hash{$dir}}[0], ${$dir_time_hash{$dir}}[1], $new or print $outerr "Could not run utime() on $new: $!\n"; + print UNDOLOG "utime ".${$dir_time_hash{$dir}}[0].", ".${$dir_time_hash{$dir}}[1].", ".$new." or print \"Could not run utime() on $new: \$!\n\""; + } elsif ($opt_parsable) { + print "utime".$del.$dir.$del.${$dir_time_hash{$dir}}[0].$del.${$dir_time_hash{$dir}}[1].$del.${$dir_time_hash{$dir}}[2].$fin_del; + } + delete $dir_time_hash{$dir}; + #print $outerr "done\n"; # debug print + } +} + +sub listvalidencodings() { + print "$_\n" for (Encode->encodings(":all")); + return 1; +} + +sub dumpoptions() { + for (keys %opts) { + s/=.*//; + print "-" if (length($_) > 1); + print "-$_\n"; + } + return 1; +} + +sub fixdouble_checkenc() { + return 1; +} + +sub char_checkenc() { + my $oldfile=shift; + my $new=$oldfile; + if ($from_is_utf8) { + if (! &$this_is_valid_utf8($new)) { + if ($opt_parsable) { + print "errormsg".$del."filenotutf8".$del.$oldfile.$fin_del; + } else { + print $outerr "this file was not validly encoded in UTF-8: \"". &$from_print($dir."/".$oldfile) ."\"\n"; + } + return undef; + } + } else { + if ($to_is_utf8 and !$opt_nosmart and &looks_like_utf8($oldfile)) { + # do nothing: e.g. from_enc is shift_jis but string is utf-8. Should + # be "smart-skipped" if to_enc is utf-8 and not produce no error here. + } + elsif (! from_to($new,$opt_f, "utf8", Encode::FB_QUIET) ) { + if ($opt_parsable) { + print "errormsg".$del."fileencodedinvalid".$del.$dir."/".$oldfile.$fin_del; + } else { + print $outerr "this file was not validly encoded in $opt_f: \"". &$from_print($dir."/".$oldfile) ."\"\n"; + } + return undef; + } + } + # $new is utf-8 now and $oldfile's encoding was valid ... + my $filenamelength; + if ($to_is_utf8) { + $new = &$norm(decode_utf8($new)); + $filenamelength=length($new); + } else { + $new=encode_utf8(NFC(decode_utf8($new))); + $filenamelength=from_to($new, "utf8", $opt_t, Encode::FB_QUIET); + } +## print "$oldfile|$utf8oldfile|$new|$filenamelength\n"; + if (! $filenamelength) { + if ($opt_parsable) { + print "errormsg".$del."charsetdoesntcoverneededcharacters".$del.$dir."/".$oldfile.$fin_del; + } else { + print $outerr "$opt_t doesn't cover all needed characters for: \"". &$from_print($dir."/".$oldfile) ."\"\n"; + } + return undef; + } elsif ($filenamelength > $maxfilenamelength) { + print $outerr "".&$from_print($dir."/".$oldfile).": resulting filename is $filenamelength bytes long (max: $maxfilenamelength)\n"; + return undef; + } + &posix_check($new); + return 1; +} + +sub printusage { + &check_for_perl_bugs; + print < + +This program comes with ABSOLUTELY NO WARRANTY; it may be copied or modified +under the terms of the GNU General Public License version 2 or 3 as published +by the Free Software Foundation. + + USAGE: convmv [options] FILE(S) +-f enc encoding *from* which should be converted +-t enc encoding *to* which should be converted +-r recursively go through directories +-i interactive mode (ask for each action) +--nfc target files will be normalization form C for UTF-8 (Linux etc.) +--nfd target files will be normalization form D for UTF-8 (OS X etc.) +--qfrom be quiet about the "from" of a rename (if it screws up your terminal e.g.) +--qto be quiet about the "to" of a rename (if it screws up your terminal e.g.) +--exec c execute command instead of rename (use #1 and #2 and see man page) +--list list all available encodings +--lowmem keep memory footprint low (see man page) +--map m apply an additional character mapping +--nosmart ignore if files already seem to be UTF-8 and convert if posible +--notest actually do rename the files +--replace will replace files if they are equal +--unescape convert%20ugly%20escape%20sequences +--upper turn to upper case +--lower turn to lower case +--parsable write a parsable todo list (see man page) +--help print this help +END +#--dotlessi care about the dotless i issue of certain locales (use with care) +} + +sub looks_like_utf8() { + my $string = shift; + if ($string =~ m/[^[:ascii:]]/ and &$this_is_valid_utf8($string)) { + return 1; + } else { + return undef; + } +} + +sub this_is_valid_utf8_decode { + my $string = shift; + if (not defined(decode_utf8($string))) { + return undef; + } else { + return 1; + } +} + +sub this_is_valid_utf8_decode_CROAK() { + my $string = shift; + # until 1.08 I used to use decode_utf8() but see perl bug #37757 (perl 5.8.7/8) + #if (not defined(decode_utf8($string)) ) { + # + # let's look for a different way to find valid utf-8 ...: + # utf8::decode() is experimental and might disappear says utf8(3pm): + #if (utf8::decode($string) != undef) { + # + # Encode::decode does not work as one might expect: + #if (Encode::decode(utf8,$string,Encode::FB_QUIET) == undef) { + # + # from_to() works for all Perl versions (at the moment ;) + # ... and here we go: with Perl 5.10 from_to(utf8..utf8) doen't work either, + # see perl bug #49830. convmv 1.10 and Perl 5.10 will again only work with + # --nosmart. + # + # okay, now perluniintro suggests to do this: + + eval 'decode_utf8($string, Encode::FB_CROAK);'; + if ($@) { + return undef; + } else { + return 1; + } +} + +sub to_ascii() { + my $a=shift; + $a =~ s/[^[:ascii:]]/?/g; + $a =~ s/[[:cntrl:]]/*/g; + return $a; +} + +sub dummy() { + return shift; +} + +sub print_ask() { # takes 2 arguments, string and askornot + if ($opt_parsable) { + return 1; + } + my $a=""; + print shift; + my $ask = shift; + while ($ask and not $a =~ m/^[yn]$/i) { + print " (y/n) "; + $a=; + } + print "\n"; + if ($a =~ m/^n$/i) { + return undef; + } else { + return 1; + } +} + +sub unescape_checkenc() { + my $name = shift; + if ($name =~ m/^[[:ascii:]]*$/) { # should we be more strict ? + &posix_check(&unescape_get_newname($name)); + return 1; + } else { + if ($opt_parsable) { + print "errormsg".$del."notanescapedfile".$del.$name.$fin_del; + } else { + print $outerr "\"",&$from_print($name),"\" not ASCII - this does not seem to be an escaped filename.\n"; + } + return undef; + } +} + +sub map_get_newname() { + $_ = shift; + return $_ if ($_ eq "." or $_ eq ".."); + _utf8_on($_); # this is needed for tr/multibyte/non-multibyte/ to work! Otherwise we would + # have to make a s/// for each character, grrr... + if ($opt_map eq "ntfs-sfm") { # see MS KB 117258 (but map : instead of / + tr/\x01-\x1f\"\*\:\<\>\?\\\|/\x{f001}-\x{f027}/; + s/ $/\x{f028}/; # Space, only if occurring as the last character of the name + s/\.$/\x{f029}/; # period, only if occurring as the last character of the name + } elsif ($opt_map eq "ntfs-sfm-undo") { + tr/\x{f001}-\x{f027}/\x01-\x1f"*:<>?\\| /; + s/\x{f028}$/ /; # Space, only if occurring as the last character of the name + s/\x{f029}$/./; # period, only if occurring as the last character of the name + } elsif ($opt_map eq "ntfs-sfu") { # +0xF000, see MS KB ???? anyone knows a link or has archived an old one? + tr/\x01-\x1f\"\*\/\<\>\?\\\|/\x{f001}-\x{f01f}\x{f022}\x{f02a}\x{f02f}\x{f03c}\x{f03e}\x{f03f}\x{f05c}\x{f07c}/; + #??? s/ $/space/; # Space, only if occurring as the last character of the name + #??? s/\.$/period/; # period, only if occurring as the last character of the name + } elsif ($opt_map eq "ntfs-sfu-undo") { + tr/\x{f001}-\x{f01f}\x{f022}\x{f02a}\x{f02f}\x{f03c}\x{f03e}\x{f03f}\x{f05c}\x{f07c}/\x01-\x1f"*\/<>?\\|/; + #??? s/space$/ /; # Space, only if occurring as the last character of the name + #??? s/period$/./; # period, only if occurring as the last character of the name + } elsif ($opt_map eq "ntfs-pretty") { + s/\"/”/g; # U+201D + s/\*/∗/g; # U+2731 + s/\?/？/g; # U+FF1F + s/\:/꞉/g; # U+A789 + s/\/＞/g; # U+FF1E + s/\|/❘/g; # U+2758 + s/\\/＼/g; # U+FF3C + } elsif ($opt_map eq "ntfs-pretty-undo") { + s/”/"/g; # U+201D + s/∗/*/g; # U+2731 + s/？/?/g; # U+FF1F + s/꞉/:/g; # U+A789 + s/＜//g; # U+FF1E + s/❘/|/g; # U+2758 + s/＼/\\/g; # U+FF3C + } else { + die "map parameter \"$opt_map\" not supported. Use one of ", + "ntfs-sfm, ntfs-sfm-undo, ", + "ntfs-sfu, ntfs-sfu-undo, ", + "ntfs-pretty, ntfs-pretty-undo\n"; + } + return $_; +} + +sub unescape_get_newname() { # return undef on error, string otherwise + my $newname = shift; +# $newname =~ s/([^a-zA-Z0-9_.-])/uc sprintf("%%%02x",ord($1))/eg; # this was done before + $newname =~ s/(%)([0-9a-fA-F][0-9a-fA-F])/chr(hex($2))/eg; + return $newname; +} + + +sub upperlower_checkenc() { + my $oldname = shift; + my $newname = upperlower_get_newname($oldname); + if (not defined($newname)) { + return undef; + } else { + &posix_check($newname); + return 1; + } +} + +sub upperlower_get_newname() { +# return undef on error, string otherwise + my $oldname = shift; + my $name=$oldname; + if (! from_to($name, $opt_f, "utf8", Encode::FB_QUIET)) { # should also leave NFD as it is ... + print $outerr "\"",&$from_print($oldname),"\" not encoded in $opt_f ? Supply the correct encoding via -f option!\n"; + return undef; + } + _utf8_on($name); # Unicode in Perl can be a real pain ... + no bytes; + if ($opt_upper) { + if ($opt_dotlessi) { + $name =~ s/ı/I/g; + $name =~ s/i/İ/g; + } + # we do not want to upper ß to SS ! Let's substitute it with + # NUL+DWSLQH (NUL may not be part of filename) and get it back after uc(). + # Unicode 5.1(draft) news: Uppercasing U+00DF (ß) LATIN SMALL LETTER SHARP S + # to the new U+1E9E LATIN CAPITAL LETTER SHARP S. + # but until now I don't see use for this in filenames ... + $name =~ s/ß/\000DWSLQH/g; + $name = uc($name); + $name =~ s/\000DWSLQH/ß/g; + } else { + if ($opt_dotlessi) { + $name =~ s/I/ı/g; + $name =~ s/İ/i/g; + } + $name = lc($name); + } + use bytes; + _utf8_off($name); + # we should also do special treatment for UTF-8 NFD of "I with dot above" in byte mode now, otherwise we get "i̇", which is a double-single dotted i ;-) + # the problems that arise with this letter are endless ... +# $name =~ s/i\314\207/i/g if ($from_is_utf8); + if (! from_to($name, "utf8", $opt_f, Encode::FB_QUIET)) { + print $outerr $opt_upper?"Upper":"Lower","case of \"",&$from_print($oldname),"\" not possible in $opt_f ! Maybe supply different encoding via -f option.\n"; + return undef; + } + return $name; +} + +sub posix_check() { + my $name=shift; + if ($name =~ m/[\000\/]/) { + print $outerr "WARNING: new filename \"",&$to_print($name),"\" contains characters, which are not POSIX filesystem conform! This may result in data loss.\n"; + $warnings_occurred=1; + } +} + +# still unused, but might be used for Netatalk CAP encoding: +sub cap2utf8() { + my $oldname = shift; + if (($oldname !~ m/^:2eDS_Store/) and ($oldname =~ /:/)) { + $oldname =~ s/(:([0-9a-f][0-9a-f]))/chr(hex($2))/eg; + } + return $oldname; +} + +sub check_for_broken_perl_release() { + # Check that most basic Perl Encode features we use work reliably + # and decide which code path we use for &this_is_valid_utf8(): + my $test = ""."\366"; + my $error = ""; + + if (not defined(decode_utf8($test))) { + $this_is_valid_utf8=\&this_is_valid_utf8_decode; + return 0; + } + $error .= "decode_utf8(\$test) check failed\n"; + + eval 'decode_utf8($test, Encode::FB_CROAK);'; + if (not $@) { + $error .= "eval 'decode_utf8(\$non-utf8, Encode::FB_CROAK);'; check failed.\n"; + } else { + $test = ""."ö"; + eval 'decode_utf8($test, Encode::FB_CROAK);'; + if ($@) { + $error .= "eval 'decode_utf8(\$utf8, Encode::FB_CROAK);'; check failed.\n"; + } else { + $this_is_valid_utf8=\&this_is_valid_utf8_decode_CROAK; + return 0; + } + } + print "Your Perl release is too broken to make convmv work reliably:\n",$error; + exit 1; +} + +sub check_for_perl_bugs() { + # Check for certain Perl fleas that we more or less have to work around: + # until 1.08 I used to use decode_utf8() but see perl bug #37757 (perl 5.8.7/8) + #if (not defined(decode_utf8($string)) ) + my $bugs = ""; + my $test = "\366"; + + my $u8test = NFD(""."ö"); # "". is intended as only so we have _utf8_off set + # otherwise from_to doesn't convert the $data to + # something else. + # print "DEBUG: string is UTF-8 flagged: ",is_utf8($u8test) ? "yes" : "no","\n"; + eval "from_to($u8test, 'utf8', 'iso-8859-1');"; + if ($u8test ne "\366") { + # Perl::Encode guys think that conversion from decomposed UTF-8 + # to any other charset does not have to be supported by from_to. + # Why, when NFC or NFD this is both perfectly valid UTF-8? + $bugs .= "#22111 "; + } + if (decode_utf8($test)) { + $bugs .= "#37757 "; + # Convmv 1.08 and below would not work here! + # Perl documentation up to 5.8.8 said that + # decode_utf8($data_that_is_not_utf_8) should return undef + } + if (! from_to($test,utf8,utf8,Encode::FB_QUIET) == undef) { + $bugs .= "#49830 "; + # convmv 1.10-1.11 would not work here! + # broken UTF-8 is silently being converted to sane UTF-8 without throwing + # an error. + } + if ($bugs) { + print "Your Perl version has fleas $bugs\n"; + } + +} diff --git a/testsuite.tar b/testsuite.tar new file mode 100644 index 0000000..63e9760 Binary files /dev/null and b/testsuite.tar differ