typo

[scripts-20-100.git] / mails / mailbox_to_maildir.pl

#!/usr/bin/perl -w
#

#
# Trouvé sur le web pour transformer un fichier mailbox en
# un dossier au format maildir
#

# $Id: mb2md.pl,v 1.26 2004/03/28 00:09:46 juri Exp $
#
# mb2md-3.20.pl      Converts Mbox mailboxes to Maildir format.
#
# Public domain.
#
# currently maintained by:
# Juri Haberland <juri@koschikode.com>
# initially wrote by:
# Robin Whittle
#
# This script's web abode is http://batleth.sapienti-sat.org/projects/mb2md/ .
# For a changelog see http://batleth.sapienti-sat.org/projects/mb2md/changelog.txt
#
# The Mbox -> Maildir inner loop is based on  qmail's script mbox2maildir, which
# was kludged by Ivan Kohler in 1997 from convertandcreate (public domain)
# by Russel Nelson.  Both these convert a single mailspool file.
#
# The qmail distribution has a maildir2mbox.c program.
#
# What is does:
# =============
#
# Reads a directory full of Mbox format mailboxes and creates a set of
# Maildir format mailboxes.  Some details of this are to suit Courier
# IMAP's naming conventions for Maildir mailboxes.
#
#   http://www.inter7.com/courierimap/
#
# This is intended to automate the conversion of the old
# /var/spool/mail/blah file - with one call of this script - and to
# convert one or more mailboxes in a specifed directory with separate
# calls with other command line arguments.
#
# Run this as the user - in these examples "blah".

# This version supports conversion of:
#
#    Date    The date-time in the "From " line of the message in the
#            Mbox format is the date when the message was *received*.
#            This is transformed into the date-time of the file which
#            contains the message in the Maildir mailbox.
#
#            This relies on the Date::Parse perl module and the utime
#            perl function.
#
#            The script tries to cope with errant forms of the
#            Mbox "From " line which it may encounter, but if
#            there is something really screwy in a From line,
#            then perhaps the script will fail when "touch"
#            is given an invalid date.  Please report the
#            exact nature of any such "From " line!
#
#
#   Flagged
#   Replied
#   Read = Seen
#   Tagged for Deletion
#
#            In the Mbox message, flags for these are found in the
#            "Status: N" or "X-Status: N" headers, where "N" is 0
#            or more of the following characters in the left column.
#
#            They are converted to characters in the right column,
#            which become the last characters of the file name,
#            following the ":2," which indicates IMAP message status.
#
#
#                F -> F      Flagged
#                A -> R      Replied
#                R -> S      Read = Seen
#                D -> T      Tagged for Deletion (Trash)
#
#            This is based on the work of Philip Mak who wrote a
#            completely separate Mbox -> Maildir converter called
#            perfect_maildir and posted it to the Mutt-users mailing
#            list on 25 December 2001:
#
#               http://www.mail-archive.com/mutt-users@mutt.org/msg21872.html
#
#            Michael Best originally integrated those changes into mb2md.
#
#
#   In addition, the names of the message files in the Maildir are of a
#   regular length and are of the form:
#
#       7654321.000123.mbox:2,xxx
#
#   Where "7654321" is the Unix time in seconds when the script was
#   run and "000123" is the six zeroes padded message number as
#   messages are converted from the Mbox file.  "xxx" represents zero or
#   more of the above flags F, R, S or T.
#
#
# ---------------------------------------------------------------------
#
#
# USAGE
# =====
#
# Run this as the user of the mailboxes, not as root.
#
#
# mb2md -h
# mb2md [-c] -m [-d destdir]
# mb2md [-c] -s sourcefile [-d destdir]
# mb2md [-c] -s sourcedir [-l wu-mailboxlist] [-R|-f somefolder] [-d destdir] [-r strip_extension]
#
#  -c            use the Content-Length: headers (if present) to find the
#                beginning of the next message
#                Use with caution! Results may be unreliable. I recommend to do
#                a run without "-c" first and only use it if you are certain,
#                that the mbox in question really needs the "-c" option
#
#  -m            If this is used then the source will
#                be the single mailbox at /var/spool/mail/blah for
#                user blah and the destination mailbox will be the
#                "destdir" mailbox itself.
#
#
#  -s source     Directory or file relative to the user's home directory,
#                which is where the the "somefolders" directories are located.
#                Or if starting with a "/" it is taken as a
#                absolute path, e.g. /mnt/oldmail/user
#
#                or
#
#                A single mbox file which will be converted to
#                the destdir.
#
#  -R            If defined, do not skip directories found in a mailbox 
#                directory, but runs recursively into each of them, 
#                creating all wanted folders in Maildir.
#                Incompatible with '-f'
#
#  -f somefolder Directories, relative to "sourcedir" where the Mbox files
#                are. All mailboxes in the "sourcedir"
#                directory will be converted and placed in the
#                "destdir" directory.  (Typically the Inbox directory
#                which in this instance is also functioning as a
#                folder for other mailboxes.)
#
#                The "somefolder" directory
#                name will be encoded into the new mailboxes' names.
#                See the examples below.
#
#                This does not save an UW IMAP dummy message file
#                at the start of the Mbox file.  Small changes
#                in the code could adapt it for looking for
#                other distinctive patterns of dummy messages too.
#
#                Don't let the source directory you give as "somefolders"
#                contain any "."s in its name, unless you want to
#                create subfolders from the IMAP user's point of
#                view.  See the example below.
#
#                Incompatible with '-f'
#
#
#  -d destdir    Directory where the Maildir format directories will be created.
#                If not given, then the destination will be ~/Maildir .
#                Typically, this is what the IMAP server sees as the
#                Inbox and the folder for all user mailboxes.
#                If this begins with a '/' the path is considered to be
#                absolute, otherwise it is relative to the users
#                home directory.
#
#  -r strip_ext  If defined this extension will be stripped from
#                the original mailbox file name before creating
#                the corresponding maildir. The extension must be
#                given without the leading dot ("."). See the example below.
#
#  -l WU-file    File containing the list of subscribed folders.  If
#                migrating from WU-IMAP the list of subscribed folders will
#                be found in the file called .mailboxlist in the users
#                home directory.  This will convert all subscribed folders
#                for a single user:
#                /bin/mb2md -s mail -l .mailboxlist -R -d Maildir
#                and for all users in a directory as root you can do the
#                following:
#                for i in *; do echo $i;su - $i -c "/bin/mb2md -s mail -l .mailboxlist -R -d Maildir";done
#
#
#  Example
#  =======
#
# We have a bunch of directories of Mbox mailboxes located at
# /home/blah/oldmail/
#
#     /home/blah/oldmail/fffff
#     /home/blah/oldmail/ggggg
#     /home/blah/oldmail/xxx/aaaa
#     /home/blah/oldmail/xxx/bbbb
#     /home/blah/oldmail/xxx/cccc
#     /home/blah/oldmail/xxx/dddd
#     /home/blah/oldmail/yyyy/huey
#     /home/blah/oldmail/yyyy/duey
#     /home/blah/oldmail/yyyy/louie
#
# With the UW IMAP server, fffff and ggggg would have appeared in the root
# of this mail server, along with the Inbox.  aaaa, bbbb etc, would have
# appeared in a folder called xxx from that root, and xxx was just a folder
# not a mailbox for storing messages.
#
# We also have the mailspool Inbox at:
#
#     /var/spool/mail/blah
#
#
# To convert these, as user blah, we give the first command:
#
#    mb2md -m
#
# The main Maildir directory will be created if it does not exist.
# (This is true of any argument options, not just "-m".)
#
#    /home/blah/Maildir/
#
# It has the following subdirectories:
#
#    /home/blah/Maildir/tmp/
#    /home/blah/Maildir/new/
#    /home/blah/Maildir/cur/
#
# Then /var/spool/blah file is read, split into individual files and
# written into /home/blah/Maildir/cur/ .
#
# Now we give the second command:
#
#    mb2md  -s oldmail -R
#
# This reads recursively all Mbox mailboxes and creates:
#
#    /home/blah/Maildir/.fffff/
#    /home/blah/Maildir/.ggggg/
#    /home/blah/Maildir/.xxx/
#    /home/blah/Maildir/.xxx.aaaa/
#    /home/blah/Maildir/.xxx.bbbb/
#    /home/blah/Maildir/.xxx.cccc/
#    /home/blah/Maildir/.xxx.aaaa/
#    /home/blah/Maildir/.yyyy/
#    /home/blah/Maildir/.yyyy.huey/
#    /home/blah/Maildir/.yyyy.duey/
#    /home/blah/Maildir/.yyyy.louie/
#
#  The result, from the IMAP client's point of view is:
#
#    Inbox -----------------
#        |
#        | fffff -----------
#        | ggggg -----------
#        |
#        - xxx -------------
#        |   | aaaa --------
#        |   | bbbb --------
#        |   | cccc --------
#        |   | dddd --------
#        |
#        - yyyy ------------
#             | huey -------
#             | duey -------
#             | louie ------
#
# Note that although ~/Maildir/.xxx/ and ~/Maildir/.yyyy may appear
# as folders to the IMAP client the above commands to not generate
# any Maildir folders of these names.  These are simply elements
# of the names of other Maildir directories. (if you used '-R', they 
# whill be able to act as normal folders, containing messages AND folders)
#
# With a separate run of this script, using just the "-s" option
# without "-f" nor "-R", it would be possible to create mailboxes which
# appear at the same location as far as the IMAP client is
# concerned.  By having Mbox mailboxes in some directory:
# ~/oldmail/nnn/ of the form:
#
#     /home/blah/oldmail/nn/xxxx
#     /home/blah/oldmail/nn/yyyyy
#
# then the command:
#
#   mb2md -s oldmail/nn
#
# will create two new Maildirs:
#
#    /home/blah/Maildir/.xxx/
#    /home/blah/Maildir/.yyyy/
#
# Then what used to be the xxx and yyyy folders now function as
# mailboxes too.  Netscape 4.77 needed to be put to sleep and given ECT
# to recognise this - deleting the contents of (Win2k example):
#
#    C:\Program Files\Netscape\Users\uu\ImapMail\aaa.bbb.ccc\
#
# where "uu" is the user and "aaa.bbb.ccc" is the IMAP server
#
# I often find that deleting all this directory's contents, except
# "rules.dat", forces Netscape back to reality after its IMAP innards
# have become twisted.  Then maybe use File > Subscribe - but this
# seems incapable of subscribing to folders.
#
# For Outlook Express, select the mail server, then click the
# "IMAP Folders" button and use "Reset list".  In the "All"
# window, select the mailboxes you want to see in normal
# usage.
#
#
# This script did not recurse subdirectories or delete old mailboxes, before addition of the '-R' parameter :)
#
# Be sure not to be accessing the Mbox mailboxes while running this
# script.  It does not attempt to lock them.  Likewise, don't run two
# copies of this script either.
#
#
# Trickier usage . . .
# ====================
#
# If you have a bunch of mailboxes in a directory ~/oldmail/doors/
# and you want them to appear in folders such as:
#
# ~/Maildir/.music.bands.doors.Jim
# ~/Maildir/.music.bands.doors.John
#
# etc. so they appear in an IMAP folder:
#
#    Inbox -----------------
#        | music
#              | bands
#                    | doors
#                          | Jim
#                          | John
#                          | Robbie
#                          | Ray
#
# Then you could rename the source directory to:
#
#  ~/oldmail/music.bands.doors/
#
# then use:
#
#   mb2md -s oldmail -f music.bands.doors
#
#
# Or simply use '-R' switch with:
#   mb2md -s oldmail -R
#
#
# Stripping mailbox extensions:
# ============================= 
#
# If you want to convert mailboxes that came for example from
# a Windows box than you might want to strip the extension of
# the mailbox name so that it won't create a subfolder in your
# mail clients view.
#
# Example:
# You have several mailboxes named Trash.mbx, Sent.mbx, Drafts.mbx
# If you don't strip the extension "mbx" you will get the following
# hierarchy:
#
# Inbox
#      |
#       - Trash 
#      |       | mbx
#      |
#       - Sent 
#      |       | mbx
#      |
#       - Drafts 
#              | mbx
#
# This is more than ugly!
# Just use:
#   mb2md -s oldmail -r mbx
#
# Note: don't specify the dot! It will be stripped off
# automagically ;)
#
#------------------------------------------------------------------------------


use strict;
use Getopt::Std;
use Date::Parse;
use IO::Handle;
use Fcntl;

                    # print the usage message
sub usage() {
    print "Usage:\n";
    print "       mb2md -h\n";
    print "       mb2md [-c] -m [-d destdir]\n";
    print "       mb2md [-c] -s sourcefile [-d destdir]\n";
    die   "       mb2md [-c] -s sourcedir [-l wu-mailboxlist] [-R|-f somefolder] [-d destdir] [-r strip_extension]\n";
}
                    # get options
my %opts;
getopts('d:f:chms:r:l:R', \%opts) || usage();
usage() if ( defined($opts{h})
        || (!defined($opts{m}) && !defined($opts{s})) );

# Get uid, username and home dir
my ($name, $passwd, $uid, $gid, $quota, $comment, $gcos, $homedir, $shell) = getpwuid($<);

# Get arguments and determine source
# and target directories.
my $mbroot = undef;     # this is the base directory for the mboxes
my $mbdir = undef;      # this is an mbox dir relative to the $mbroot
my $mbfile = undef;     # this is an mbox file
my $dest = undef;
my $strip_ext = undef;
my $use_cl = undef;     # defines whether we use the Content-Length: header if present

# if option "-c" is given, we use the Content-Length: header if present
# dangerous! may be unreliable, as the whole CL stuff is a bad idea
if (defined($opts{c}))
{
        $use_cl = 1;
} else {
        $use_cl = 0;
}

# first, if the user has gone the -m option
# we simply convert their mailfile
if (defined($opts{m}))
{
        if (defined($ENV{'MAIL'})) {
                $mbfile = $ENV{'MAIL'};
        } elsif ( -f "/var/spool/mail/$name" ) {
                $mbfile = "/var/spool/mail/$name"
        } elsif ( -f "/var/mail/$name" ) {
                $mbfile = "/var/mail/$name"
        } else {
                die("I searched \$MAIL, /var/spool/mail/$name and /var/mail/$name, ".
                        "but I couldn't find your mail spool file - ");
        }
}
# see if the user has specified a source directory
elsif (defined($opts{s}))
{
        # if opts{s} doesn't start with a "/" then
        # it is a subdir of the users $home
        # if it does start with a "/" then
        # let's take $mbroot as a absolut path
        $opts{s} = "$homedir/$opts{s}" if ($opts{s} !~ /^\//); 

        # check if the given source is a mbox file
        if (-f $opts{s})
        {
                $mbfile = $opts{s};
        }

        # otherwise check if it is a directory
        elsif (-d $opts{s})
        {
                $mbroot = $opts{s};
                # get rid of trailing /'s
                $mbroot =~ s/\/$//;

                # check if we have a specified sub directory,
                # otherwise the sub directory is '.'
                if (defined($opts{f}))
                {
                        $mbdir = $opts{f};
                        # get rid of trailing /'s
                        $mbdir =~ s/\/$//;
                }
        }

        # otherwise we have an error
        else
        {
                die("Fatal: Source is not an mbox file or a directory!\n");
        }
}


# get the dest
defined($opts{d}) && ($dest = $opts{d}) || ($dest = "Maildir");
# see if we have anything to strip
defined($opts{r}) && ($strip_ext = $opts{r});
# No '-f' with '-R'
if((defined($opts{R}))&&(defined($opts{f}))) { die "No recursion with \"-f\"";}
# Get list of folders
my @flist;
if(defined($opts{l}))
{
    open (LIST,$opts{l}) or die "Could not open mailbox list $opts{l}: $!";
    @flist=<LIST>;
    close LIST;
}

# if the destination is relative to the home dir,
# check that the home dir exists
die("Fatal: home dir $homedir doesn't exist.\n") if ($dest !~ /^\// &&  ! -e $homedir);

#
# form the destination value
# slap the home dir on the front of the dest if the dest does not begin
# with a '/'
$dest = "$homedir/$dest" if ($dest !~ /^\//);
# get rid of trailing /'s
$dest =~ s/\/$//;


# Count the number of mailboxes, or
# at least files, we found.
my $mailboxcount = 0;

# Since we'll be making sub directories of the main
# Maildir, we need to make sure that the main maildir
# exists
&maildirmake($dest);

# Now we do different things depending on whether we convert one mbox
# file or a directory of mbox files
if (defined($mbfile))
{
        if (!isamailboxfile($mbfile))
        {
              print "Skipping $mbfile: not a mbox file\n";
        }
        else
        {
              print "Converting $mbfile to maildir: $dest\n";
              # this is easy, we just run the convert function
              &convert($mbfile, $dest);
        }
}
# if '-f' was used ...
elsif (defined($mbdir))
{
        print "Converting mboxdir/mbdir: $mbroot/$mbdir to maildir: $dest/\n";
        
        # Now set our source directory
        my $sourcedir = "$mbroot/$mbdir";

        # check that the directory we are supposed to be finding mbox
        # files in, exists and is a directory
        -e $sourcedir or die("Fatal: MBDIR directory $sourcedir/ does not exist.\n");
        -d $sourcedir or die("Fatal: MBDIR $sourcedir is not a directory.\n");

        
        &convertit($mbdir,"");
}
# Else, let's work in $mbroot
else
{
        opendir(SDIR, $mbroot)
                or die("Fatal: Cannot open source directory $mbroot/ \n");


        while (my $sourcefile = readdir(SDIR))
        {
                if (-d "$mbroot/$sourcefile") {
                        # Recurse only if requested (to be changed ?)
                        if (defined($opts{R})) {
                                print "convertit($sourcefile,\"\")\n";
                                &convertit($sourcefile,"");
                        } else {
                        print("$sourcefile is a directory, but '-R' was not used... skipping\n");
                        }
                }
                elsif (!-f "$mbroot/$sourcefile")
                {
                        print "Skipping $mbroot/$sourcefile : not a file nor a dir\n";
                        next;
                }
                elsif (!isamailboxfile("$mbroot/$sourcefile"))
                {
                        print "Skipping $mbroot/$sourcefile : not a mbox file\n";
                        next;
                }
                else 
                {
                        &convertit($sourcefile,"");
                }
        } # end of "while ($sfile = readdir(SDIR))" loop.
        closedir(SDIR);
        printf("$mailboxcount files processed.\n");
}
#

exit 0;

# My debbugging placeholder I can put somewhere to show how far the script ran.
# die("So far so good.\n\n");

# The isamailboxfile function
# ----------------------
# 
# Here we check if the file is a mailbox file, not an address-book or 
# something else.
# If file is empty, we say it is a mbox, to create it empty.
#
# Returns 1 if file is said mbox, 0 else.
sub isamailboxfile {
        my ($mbxfile) = @_;
        return 1 if(-z $mbxfile);
        sysopen(MBXFILE, "$mbxfile", O_RDONLY) or die "Could not open $mbxfile ! \n";
        while(<MBXFILE>) {
                if (/^From/) {
                        close(MBXFILE);
                        return 1;
                }
                else {
                        close(MBXFILE);
                        return 0;
                }
        }
}

# The convertit function
# -----------------------
#
# This function creates all subdirs in maildir, and calls convert() 
# for each mbox file.
# Yes, it becomes the 'main loop' :)
sub convertit
{
        # Get subdir as argument
        my ($dir,$oldpath) = @_;
        
        $oldpath =~ s/\/\///;

        # Skip files beginning with '.' since they are
        # not normally mbox files nor dirs (includes '.' and '..')
        if ($dir =~ /^\./)
        {
                print "Skipping $dir : name begins with a '.'\n";
                return;
        }
        my $destinationdir = $dir;
        my $temppath = $oldpath;

        # We don't want to have .'s in the $targetfile file
        # name because they will become directories in the
        # Maildir. Therefore we convert them to _'s
        $temppath =~ s/\./\_/g;
        $destinationdir =~ s/\./\_/g;
        
        # Appending $oldpath => path is only missing $dest
        $destinationdir = "$temppath.$destinationdir";

        # Converting '/' to '.' in $destinationdir
        $destinationdir =~s/\/+/\./g;
        
        # source dir
        my $srcdir="$mbroot/$oldpath/$dir";

        printf("convertit(): Converting $dir in $mbroot/$oldpath to $dest/$destinationdir\n");
        &maildirmake("$dest/$destinationdir");
        print("destination = $destinationdir\n");
        if (-d $srcdir) {
                opendir(SUBDIR, "$srcdir") or die "can't open $srcdir !\n";
                my @subdirlist=readdir(SUBDIR);
                closedir(SUBDIR);
                foreach (@subdirlist) {
                        next if (/^\.+$/);
                        print("Sub: $_\n");
                        print("convertit($_,\"$oldpath/$dir\")\n");
                        &convertit($_,"$oldpath/$dir");
                } 
        } else {
                # Source file verifs ....
                #
                return if(defined($opts{l}) && !inlist("$oldpath/$dir",@flist));

                if (!isamailboxfile("$mbroot/$oldpath/$dir"))
                {
                        print "Skipping $dir (is not mbox)\n";
                        next;
                }

                # target file verifs...
                #
                # if $strip_extension is defined,
                # strip it off the $targetfile
                defined($strip_ext) && ($destinationdir =~ s/\.$strip_ext$//);
                &convert("$mbroot/$oldpath/$dir","$dest/$destinationdir");
                $mailboxcount++;
        }
}
# The maildirmake function
# ------------------------
#
# It does the same thing that the maildirmake binary that 
# comes with courier-imap distribution
#
sub maildirmake
{
        foreach(@_) {
                -d $_ or mkdir $_,0700 or die("Fatal: Directory $_ doesn't exist and can't be created.\n");
        
                -d "$_/tmp" or mkdir("$_/tmp",0700) or die("Fatal: Unable to make $_/tmp/ subdirectory.\n");
                -d "$_/new" or mkdir("$_/new",0700) or die("Fatal: Unable to make $_/new/ subdirectory.\n");
                -d "$_/cur" or mkdir("$_/cur",0700) or die("Fatal: Unable to make $_/cur/ subdirectory.\n");
        }
}

# The inlist function
# ------------------------
#
# It checks that the folder to be converted is in the list of subscribed
# folders in WU-IMAP
#
sub inlist
{
        my ($file,@flist) = @_;
        my $valid = 0;
        # Get rid of the first / if any
        $file =~ s/^\///;
        foreach my $folder (@flist) {
                chomp $folder;
                if ($file eq $folder) {
                        $valid = 1;
                        last;
                }
        }
        if (!$valid) {
                print "$file is not in list\n";
        }
        else {
                print "$file is in list\n";
        }

        return $valid;
}
        
# 

# The convert function
# ---------------------
#
# This function does the down and dirty work of
# actually converting the mbox to a maildir
#
sub convert
{
        # get the source and destination as arguments
        my ($mbox, $maildir) = @_;

        printf("Source Mbox is $mbox\n");
        printf("Target Maildir is $maildir \n") ;

        # create the directories for the new maildir
        #
        # if it is the root maildir (ie. converting the inbox)
        # these already exist but thats not a big issue

        &maildirmake($maildir);

        # Change to the target mailbox directory.

        chdir "$maildir" ;

                    # Converts a Mbox to multiple files
                    # in a Maildir.
                    # This is adapted from mbox2maildir.
                    #
                    # Open the Mbox mailbox file.


        if (sysopen(MBOX, "$mbox", O_RDONLY))
        {
            #printf("Converting Mbox   $mbox . . .  \n");
        }
        else
        {
            die("Fatal: unable to open input mailbox file: $mbox ! \n");
        }

                    # This loop scans the input mailbox for
                    # a line starting with "From ".  The
                    # "^" before it is pattern-matching
                    # lingo for it being at the start of a
                    # line.
                    #
                    # Each email in Mbox mailbox starts
                    # with such a line, which is why any
                    # such line in the body of the email
                    # has to have a ">" put in front of it.
                    #
                    # This is not required in a Maildir
                    # mailbox, and some majik below
                    # finds any such quoted "> From"s and
                    # gets rid of the "> " quote.
                    #
                    # Each email is put in a file
                    # in the cur/ subdirectory with a
                    # name of the form:
                    #
                    #    nnnnnnnnn.cccc.mbox:2,XXXX
                    #
                    # where:
                    #    "nnnnnnnnn" is the Unix time since
                    #       1970 when this script started
                    #       running, incremented by 1 for
                    #       every email.  This is to ensure
                    #       unique names for each message
                    #       file.
                    #
                    #    ".cccc" is the message count of
                    #       messages from this mbox.
                    #
                    #    ".mbox" is just to indicate that
                    #       this message was converted from
                    #       an Mbox mailbox.
                    #
                    #    ":2," is the start of potentially
                    #       multiple IMAP flag characters
                    #       "XXXX", but may be followed by
                    #       nothing.
                    #
                    # This is sort-of  compliant with
                    # the Maildir naming conventions
                    # specified at:
                    #
                    # http://www.qmail.org/man/man5/maildir.html
                    #
                    # This approach does not involve the
                    # process ID or the hostname, but it is
                    # probably good enough.
                    #
                    # When the IMAP server looks at this
                    # mailbox, it will move the files to
                    # the cur/ directory and change their
                    # names as it pleases.  In the case
                    # of Courier IMAP, the names will
                    # become like:
                    #
                    #   995096541.25351.mbox:2,S
                    #
                    # with 25351 being Courier IMAP's
                    # process ID.  The :2, is the start
                    # of the flags, and the "S" means
                    # that this one has been seen by
                    # the user.  (But is this the same
                    # meaning as the user actually
                    # having opened the message to see
                    # its contents, rather than just the
                    # IMAP server having been asked to
                    # list the message's Subject etc.
                    # so the client could list it in the
                    # visible Inbox?)
                    #
                    # This contrasts with a message
                    # created by Courier IMAP, say with
                    # a message copy, which is like:
                    #
                    #   995096541.25351.zair,S=14285:2,S
                    #
                    # where ",S=14285" is the size of the
                    # message in bytes.
                    #
                    # Courier Maildrop's names are similar
                    # but lack the ":2,XXXX" flags . . .
                    # except for my modified Maildrop
                    # which can deliver them with a
                    # ":2,T" - flagged for deletion.
                    #
                    # I have extended the logic of the
                    # per-message inner loop to stop
                    # saving a file for a message with:
                    #
                    # Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA
                    #
                    # This is the dummy message, always
                    # at the start of an Mbox format
                    # mailbox file - and is put there
                    # by UW IMAPD.  Since quite a few
                    # people will use this for
                    # converting from a UW system,
                    # I figure it is worth it.
                    #
                    # I will not save any such message
                    # file for the dummy message.
                    #
                    # Plan
                    # ----
                    #
                    # We want to read the entire Mbox file, whilst
                    # going through a loop for each message we find.
                    #
                    # We want to read all the headers of the message,
                    # starting with the "From " line.   For that "From "
                    # line we want to get a date.
                    #
                    # For all other header lines, we want to store them
                    # in $headers whilst parsing them to find:
                    #
                    #   1 - Any flags in the "Status: " or "X-Status: " or
                    #       "X-Mozilla-Status: " lines.
                    #
                    #   2 - A subject line indicating this is the dummy message
                    #       at the start (typically, but not necessarily) of
                    #       the Mbox.
                    #
                    # Once we reach the end of the headers, we will crunch any
                    # flags we found to create a file name.  Then, unless this is
                    # the dummy message we create that file and write all the
                    # headers to it.
                    #
                    # Then we continue reading the Mbox, converting ">From " to
                    # "From " and writing it to the file, until we reach one of:
                    #
                    #   1 - Another "From " line (indicating the start of another
                    #       message).
                    #
                    # or
                    #
                    #   2 - The end of the Mbox.
                    #
                    # In the former case, which we detect at the start of the loop
                    # we need to close the file and touch it to alter its date-time.
                    #
                    # In the later case, we also need to close the file and touch
                    # it to alter its date-time - but this is beyond the end of the
                    # loop.


                    # Variables
                    # ---------

        my $messagecount = 0;

                    # For generating unique filenames for
                    # each message.  Initialise it here with
                    # numeric time in seconds since 1970.
        my $unique = time;

                    # Name of message file to delete if we found that
                    # it was created by reading the Mbox dummy message.

        my $deletedummy = '';

                    # To store the complete "From (address) (date-time)
                    # which delineates the start of each message
                    # in the Mbox
        my $fromline = '';


                    # Set to 1 when we are reading the header lines,
                    # including the "From " line.
                    #
                    # 0 means we are reading the message body and looking
                    # for another "From " line.

        my $inheaders = 0;

                    # Variable to hold all headers (apart from
                    # the first line "From ...." which is not
                    # part of the message itself.
        my $headers = '';

                    # Variable to hold the accumulated characters
                    # we find in header lines of the type:
                    #
                    #    Status:
                    #    X-Status:
                    #    X-Mozilla-Status:
                    #    X-Evolution:
        my $flags = '';

                    # To build the file name for the message in.
        my $messagefn = '';


                    # The date string from the "From " line of each
                    # message will be written here - and used by
                    # touch to alter the date-time of each message
                    # file.  Put non-date text here to make it
                    # spit the dummy if my code fails to find a
                    # date to write into this.

        my $receivedate = 'Bogus';

        # The subject of the message
        my $subject = '';

        my $previous_line_was_empty = 1;

                    # We record the message start line here, for error
                    # reporting.
        my $startline;

                    # If defined, we use this as the number of bytes in the
                    # message body rather than looking for a /^From / line.
        my $contentlength;

                            # A From lines can either occur as the first
                            # line of a file, or after an empty line.
                            # Most mail systems will quote all From lines
                            # appearing in the message, but some will only
                            # do it when necessary.
                            # Since we initialise the variable to true,
                            # we don't need to check for beginning of file.

        while(<MBOX>)
        {
                            # exchange possible Windows EOL (CRLF) with Unix EOL (LF)
            $_ =~ s/\r\n$/\n/;

            if ( /^From /
                && $previous_line_was_empty
                && (!defined $contentlength) 
               )
            {
                            # We are reading the "From " line which has an
                            # email address followed by a receive date.
                            # Turn on the $inheaders flag until we reach
                            # the end of the headers.

                $inheaders = 1;

                            # record the message start line

                $startline = $.;

                            # If this is not the first run through the loop
                            # then this means we have already been working
                            # on a message.

                if ($messagecount > 0)
                {
                            # If so, then close that message file and then
                            # use utime to change its date-time.
                            #
                            # Note this code should be duplicated to do
                            # the same thing at the end of the while loop
                            # since we must close and touch the final message
                            # file we were writing when we hit the end of the
                            # Mbox file.

                    close (OUT);
                    if ($messagefn ne '') {
                        my $t = str2time($receivedate);
                        utime $t, $t, $messagefn;
                    }
                }

                            # Because we opened the Mbox file without any
                            # variable, I think this means that we have its
                            # current line in Perl's default variable "$_".
                            # So all sorts of pattern matching magic works
                            # directly on it.

                            # We are currently reading the first line starting with
                            # "From " which contains the date we want.
                            #
                            # This will be of the form:
                            #
                            #     From dduck@test.org Wed Nov 24 11:05:35 1999
                            #
                            # at least with UW-IMAP.
                            #
                            # However, I did find a nasty exception to this in my
                            # tests, of the form:
                            #
                            #   "bounce-MusicNewsletter 5-rw=test.org"@announce2.mp3.com
                            #
                            # This makes it trickier to get rid of the email address,
                            # but I did find a way.  I can't rule out that there would
                            # be some address like this with an "@" in the quoted
                            # portion too.
                            #
                            # Unfortunately, testing with an old Inbox Mbox file,
                            # I also found an instance where the email address
                            # had no @ sign at all.  It was just an email
                            # account name, with no host.
                            #
                            # I could search for the day of the week.  If I skipped
                            # at least one word of non-whitespace (1 or more contiguous
                            # non-whitespace characters) then searched for a day of
                            # the week, then I should be able to avoid almost
                            # every instance of a day of the week appearing in
                            # the email address.
                            #
                            # Do I need a failsafe arrangement to provide some
                            # other date to touch if I don't get what seems like
                            # a date in my resulting string?  For now, no.
                            #
                            # I will take one approach if there is an @ in the
                            # "From " line and another (just skip the first word
                            # after "From ") if there is no @ in the line.
                            #
                            # If I knew more about Perl I would probably do it in
                            # a more elegant way.

                            # Copy the current line into $fromline.

                $fromline = $_;

                            # Now get rid of the "From ". " =~ s" means substitute.
                            # Find the word "From " at the start of the line and
                            # replace it with nothing.  The nothing is what is
                            # between the second and third slash.

                $fromline =~ s/^From // ;


                            # Likewise get rid of the email address.
                            # This first section is if we determine there is one
                            # (or more . . . ) "@" characters in the line, which
                            # would normally be the case.

                if ($fromline =~ m/@/)
                {
                            # The line has at least one "@" in it, so we assume
                            # this is in the middle of an email address.
                            #
                            # If the email address had no spaces, then we could
                            # get rid of the whole thing by searching for any number
                            # of non-whitespace characters (\S) contiguously, and
                            # then I think a space.  Subsitute nothing for this.
                            #
                            #    $fromline =~ s/(\S)+ //    ;
                            #
                            # But we need something to match any number of non-@
                            # characters, then the "@" and then all the non-whitespace
                            # characters from there (which takes us to the end of
                            # "test.org") and then the space following that.
                            #
                            # A tutorial on regular expressions is:
                            #
                            #    http://www.perldoc.com/perl5.6.1/pod/perlretut.html
                            #
                            # Get rid of all non-@ characters up to the first "@":

                    $fromline =~ s/[^@]+//;


                            # Get rid of the "@".

                    $fromline =~ s/@//;
                }
                            # If there was an "@" in the line, then we have now
                            # removed the first one (lets hope there aren't more!)
                            # and everything which preceded it.
                            #
                            # we now remove either something like
                            # '(foo bar)'. eg. '(no mail address)',
                            # or everything after the '@' up to the trailing
                            # timezone
                            #
                            # FIXME: all those regexp should be combined to just one single one

                $fromline =~ s/(\((\S*| )+\)|\S+) *//;

                chomp $fromline;

                            # Stash the date-time for later use.  We will use it
                            # to touch the file after we have closed it.

                $receivedate = $fromline;
                
                            # Debugging lines:
                            #
                            # print "$receivedate is the receivedate of message $messagecount.\n";
                            # $receivedate = "Wed Nov 24 11:05:35 1999";
                            #
                            # To look at the exact date-time of files:
                            #
                            #   ls -lFa --full-time
                            #
                            # End of handling the "From " line.
            }


                            # Now process header lines which are not the "From " line.

            if (    ($inheaders eq 1)
                 && (! /^From /)
               )
            {
                            # Now we are reading the header lines after the "From " line.
                            # Keep looking for the blank line which indicates the end of the
                            # headers.


                            # ".=" means append the current line to the $headers
                            # variable.
                            #
                            # For some reason, I was getting two blank lines
                            # at the end of the headers, rather than one,
                            # so I decided not to read in the blank line
                            # which terminates the headers.
                            #
                            # Delete the "unless ($_ eq "\n")" to get rid
                            # of this kludge.

                $headers .= $_ unless ($_ eq "\n");

                            # Now scan the line for various status flags
                            # and to fine the Subject line.

                $flags  .= $1 if /^Status: ([A-Z]+)/;
                $flags  .= $1 if /^X-Status: ([A-Z]+)/;
                if (/^X-Mozilla-Status: ([0-9a-f]{4})/i)
                {
                  $flags .= 'R' if (hex($1) & 0x0001);
                  $flags .= 'A' if (hex($1) & 0x0002);
                  $flags .= 'D' if (hex($1) & 0x0008);
                }
                if(/^X\-Evolution:\s+\w{8}\-(\w{4})/oi)
                {
                    $b = pack("H4", $1); #pack it as 4 digit hex (0x0000)
                    $b = unpack("B32", $b); #unpack into bit string

                    # "usually" only the right most six bits are used
                    # however, I have come across a seventh bit in
                    # about 15 (out of 10,000) messages with this bit
                    # activated.
                    # I have not found any documentation in the source.
                    # If you find out what it does, please let me know.

                    # Notes:
                    #   Evolution 1.4 does mark forwarded messages.
                    #   The sixth bit is to denote an attachment

                    $flags .= 'A' if($b =~ /[01]{15}1/); #replied
                    $flags .= 'D' if($b =~ /[01]{14}1[01]{1}/); #deleted
                    $flags .= 'T' if($b =~ /[01]{13}1[01]{2}/); #draft
                    $flags .= 'F' if($b =~ /[01]{12}1[01]{3}/); #flagged
                    $flags .= 'R' if($b =~ /[01]{11}1[01]{4}/); #seen/read
                }
                $subject = $1 if /^Subject: (.*)$/;
                if ($use_cl eq 1)
                {
                        $contentlength = $1 if /^Content-Length: (\d+)$/;
                }

                            # Now look out for the end of the headers - a blank
                            # line.  When we find it, create the file name and
                            # analyse the Subject line.

                if ($_ eq "\n")
                {
                            # We are at the end of the headers.  Set the
                            # $inheaders flag back to 0.

                    $inheaders = 0;

                            # Include the current newline in the content length

                    ++$contentlength if defined $contentlength;

                            # Create the file name for the current message.
                            #
                            # A simple version of this would be:
                            #
                            #   $messagefn = "cur/$unique.$messagecount.mbox:2,";
                            #
                            # This would create names with $messagecount values of
                            # 1, 2, etc.  But for neatness when looking at a
                            # directory of such messages, sorted by filename,
                            # I want to have leading zeroes on message count, so
                            # that they would be 000001 etc.  This makes them
                            # appear in message order rather than 1 being after
                            # 19 etc.  So this is good for up to 999,999 messages
                            # in a mailbox.  It is a cosmetic matter for a person
                            # looking into the Maildir directory manually.
                            # To do this, use sprintf instead with "%06d" for
                            # 6 characters of zero-padding:

                        $messagefn = sprintf ("cur/%d.%06d.mbox:2,", $unique, $messagecount) ;


                            # Append flag characters to the end of the
                            # filename, according to flag characters
                            # collected from the message headers

                    $messagefn .= 'F' if $flags =~ /F/; # Flagged.
                    $messagefn .= 'R' if $flags =~ /A/; # Replied to.
                    $messagefn .= 'S' if $flags =~ /R/; # Seen or Read.
                    $messagefn .= 'T' if $flags =~ /D/; # Tagged for deletion.


                            # Opens filename $messagefn for output (>) with filehandle OUT.

                    open(OUT, ">$messagefn") or die("Fatal: unable to create new message $messagefn");

                            # Count the messages.

                    $messagecount++;

                            # Only for the first message,
                            # check to see if it is a dummy.
                            # Delete the message file we
                            # just created if it was for the
                            # dummy message at the start
                            # of the Mbox.
                            #
                            # Add search terms as required.
                            # The last 2 lines are for rent.
                            #
                            # "m" means match the regular expression,
                            # but we can do without it.
                            #
                            # Do I need to escape the ' in "DON'T"?
                            # I didn't in the original version.

                    if (   (($messagecount == 1) && defined($subject))
                        && ($subject =~ m/^DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA/)
                       )
                    {
                            # Stash the file name of the dummy message so we
                            # can delete it later.

                        $deletedummy = "$messagefn";
                    }

                            # Print the collected headers to the message file.

                    print OUT "$headers";


                            # Clear $headers and $flags ready for the next message.

                    $headers = '';
                    $flags = '';

                            # End of processing the headers once we found the
                            # blank line which terminated them
                }

                            # End of dealing with the headers.
            }


            if ( $inheaders eq 0)
            {

                            # We are now processing the message body.
                            #
                            # Now we have passed the headers to the
                            # output file, we scan until the while
                            # loop finds another "From " line.

                            # Decrement our content length if we're
                            # using it to find the end of the message
                            # body

                if (defined $contentlength) {

                            # Decrement our $contentlength variable

                    $contentlength -= length($_);

                            # The proper end for a message with Content-Length
                            # specified is the $contentlength variable should
                            # be exactly -1 and we should be on a bare
                            # newline.  Note that the bare newline is not
                            # printed to the end of the current message as
                            # it's actually a message separator in the mbox
                            # format rather than part of the message.  The
                            # next line _should_ be a From_ line, but just in
                            # case the Content-Length header is incorrect
                            # (e.g. a corrupt mailbox), we just continue
                            # putting lines into the current message until we
                            # see the next From_ line.

                    if ($contentlength < 0) {
                        if ($contentlength == -1 && $_ eq "\n") {
                            $contentlength = undef;
                            next;
                        }
                        $contentlength = undef;
                    }
                }

                            #
                            # We want to copy every part of the message
                            # body to the output file, except for the
                            # quoted ">From " lines, which was the
                            # way the IMAP server encoded body lines
                            # starting with "From ".
                            #
                            # Pattern matching Perl majik to
                            # get rid of an Mbox quoted From.
                            #
                            # This works on the default variable "$_" which
                            # contains the text from the Mbox mailbox - I
                            # guess this is the case because of our
                            # (open(MBOX ....) line above, which did not
                            # assign this to anything else, so it would go
                            # to the default variable.  This enables
                            # inscrutably terse Perlisms to follow.
                            #
                            # "s" means "Subsitute" and it looks for any
                            # occurrence of ">From" starting at the start
                            # of the line.  When it finds this, it replaces
                            # it with "From".
                            #
                            # So this finds all instances in the Mbox message
                            # where the original line started with the word
                            # "From" but was converted to ">From" in order to
                            # not be mistaken for the "From ..." line which
                            # is used to demark each message in the Mbox.
                            # This was was a destructive conversion because
                            # any message which originally had ">From" at the
                            # start of the line, before being put into the
                            # Mbox, will now have that line without the ">".

                s/^>From /From /;

                            # Glorious tersness here.  Thanks Simon for
                            # explaining this.
                            #
                            # "print OUT" means print the default variable to
                            # the file of file handle OUT.  This is where
                            # the bulk of the message text is written to
                            # the output file.

                print OUT or die("Fatal: unable to write to new message to $messagefn");


                            # End of the if statement dealing with message body.
            }

            $previous_line_was_empty = ( $_ eq "\n" );

                            # End of while (MBOX) loop.
        }
                            # Close the input file.

        close(MBOX);

                            # Close the output file, and duplicate the code
                            # from the start of the while loop which touches
                            # the date-time of the most recent message file.

        close(OUT);
        if ($messagefn ne '') {
            my $t = str2time($receivedate);
            utime $t, $t, $messagefn;
        }

                            # After all the messages have been
                            # converted, check to see if the
                            # first one was a dummy.
                            # if so, delete it and make
                            # the message count one less.

        if ($deletedummy ne "")
        {
            printf("Dummy mail system first message detected and not saved.\n");
            unlink $deletedummy;

            $messagecount--;

        }

        printf("$messagecount messages.\n\n");
}