Free Support Forum - aspose.com

Missing Message When Adding to PST

Using 20.3-java version

We have an in issue where a Message is not added to a pst file we created. The attached code was run with the following arguments.

--pstFile blah.pst --count 31000 --verify --seed 6360491446827453733

package message_archiver.apps.pst;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Random;

import org.apache.commons.lang.RandomStringUtils;

import com.aspose.email.Attachment;
import com.aspose.email.FileFormatVersion;
import com.aspose.email.FolderInfo;
import com.aspose.email.FolderInfoCollection;
import com.aspose.email.License;
import com.aspose.email.MailAddress;
import com.aspose.email.MailAddressCollection;
import com.aspose.email.MailMessage;
import com.aspose.email.MapiConversionOptions;
import com.aspose.email.MapiMessage;
import com.aspose.email.PersonalStorage;
import com.aspose.email.SaveOptions;


/**
 * 
 * An application to create a PST with missing message bug
 * The source code will be sent to aspose team
 *
 */
public class PstMissingMessageGeneratorApp {

    private static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8");

    private final File pstFile;
    private final Random random;
    private final int maxMessages;
    private final boolean verifyMessage;
    private final File outputDir;
    
    private final List<String> mimeTypes = Arrays.asList("application/octet-stream",
            "application/pgp-signature",
            "application/pkcs7-signature",
            "text/plain",
            "text/html",
            "text/calendar",
            "message/rfc822"); 
    
    public PstMissingMessageGeneratorApp(File pstFile, int maxMessages, boolean verifyMessage, File outputDir, long seed) {
        Random seedGenerator = new Random();
        long realSeed = seed == 0 ? seedGenerator.nextLong() : seed;
        System.out.println(
                "Running with pstFile:" + pstFile.getAbsolutePath() + " maxMessages:" + maxMessages
                        + " verify:" + verifyMessage + " outputDir:" + outputDir + " seed:" + realSeed);
        this.random = new Random(realSeed);
        this.pstFile = pstFile;
        this.maxMessages = maxMessages;
        this.verifyMessage = verifyMessage;
        this.outputDir = outputDir;
    }
    
    private void run() {
        PersonalStorage pst = null;
        int msgCount = 0;
        long timeStart = System.currentTimeMillis();
        
        try {
            pst = PersonalStorage.create(pstFile.getAbsolutePath(), FileFormatVersion.Unicode);
            FolderInfo pstFolder = pst.getRootFolder().addSubFolder("Data");
            
            while(msgCount < maxMessages) {
                try {
                    MapiMessage message = createRandomMessage(msgCount);
                    pstFolder.addMessage(message);
                    msgCount++;
                    
                    if(msgCount % 100 == 0) {
                        printStats(timeStart, msgCount);
                    }
                    
                    if (msgCount % 1000 == 0) {
                        pstFolder = pst.getRootFolder().addSubFolder("Data" + msgCount);
                    }
                    
                } catch(Exception ex) {
                    System.out.println("Failed for msg:" + msgCount);
                    ex.printStackTrace();
                }
            }
            printStats(timeStart, msgCount);

            if (verifyMessage) {
                FolderInfo rootFolder = pst.getRootFolder();
                FolderInfoCollection subFolders = rootFolder.getSubFolders();
                for (FolderInfo subFolder : subFolders) {
                    int count = subFolder.getContentCount();
                    int realCount = subFolder.getContents().size();
                    if (realCount != count) {
                        System.out.println("final expected: " + count + ", final real: " + realCount
                                + " for folder:" + subFolder.getDisplayName());
                    } else {
                        System.out.println(
                                "Counts are equal for folder:" + subFolder.getDisplayName());
                    }
                }
            }

        } finally {
            if(pst != null) {
                pst.dispose();
            }
        }
        
        printStats(timeStart, msgCount);
    }
    
    private void printStats(long timeStart, int msgCount) {
        long timeUsed = System.currentTimeMillis() - timeStart;
        System.out.println("Total: " + msgCount + "\tTime: " + timeUsed + "\tSpeed: " + getSpeed(timeUsed, msgCount));
    }
    
    private String getSpeed(long timeUsed, long count) {
        if(timeUsed < 1) {
            return "0/s";
        }
        return String.format("%.2f/s", 1000.0 * count / timeUsed);
    }
    
    private MapiMessage createRandomMessage(int msgCount) {
        MailMessage message = new MailMessage();
        //message.add
        message.setBodyEncoding(DEFAULT_CHARSET);
        
        message.setFrom(getRandomMailAddress());
        message.setTo(getRandomMailAddressCollection());
        message.setCC(getRandomMailAddressCollection());
        message.setBcc(getRandomMailAddressCollection());
        
        message.setSubject(msgCount + ":" + RandomStringUtils.random(random.nextInt(100), 0, 0, true, false, null, random));
        message.setBody(RandomStringUtils.random(random.nextInt(10000), 0, 0, false, false, null, random));
        message.setDate(getRandomDate());
        
        int attCount = random.nextInt(10) > 0 ? random.nextInt(2) + 1 : 0;
        for(int i=0; i<attCount; i++) {
            message.addAttachment(getRandomAttachment());
        }
        
        if (outputDir != null) {
            message.save((new File(outputDir, msgCount + ".eml")).getAbsolutePath(), SaveOptions.getDefaultEml());
        }
        
        MapiMessage mapiMsg = MapiMessage.fromMailMessage(message, MapiConversionOptions.getUnicodeFormat());
        
        return mapiMsg;
    }
    
    private Date getRandomDate() {
        return new Date(3600L * 24L * random.nextInt(3600 * 24 * 365 * 45));
    }
    
    private Attachment getRandomAttachment() {
        String fileName = RandomStringUtils.random(random.nextInt(30), 0, 0, true, false, null, random);
        String strType = mimeTypes.get(random.nextInt(mimeTypes.size()));
        
        byte[] bytes = RandomStringUtils.random(random.nextInt(10240), 0, 0, true, false, null, random).getBytes(DEFAULT_CHARSET);
        ByteArrayInputStream stream = new ByteArrayInputStream(bytes);
        return new Attachment(stream, fileName,  strType);
    }
    
    private MailAddressCollection getRandomMailAddressCollection() {
        MailAddressCollection ac = new MailAddressCollection();
        int count = random.nextInt(10); // Add lots of addresses
        for(int i=0; i<count; i++) {
            ac.addMailAddress(getRandomMailAddress());
        }        
        return ac;
    }
    
    private MailAddress getRandomMailAddress() {
        
        String addr =
                RandomStringUtils.random(random.nextInt(50) + 1, 0, 0, true, false, null, random) + '@' + 
                RandomStringUtils.random(random.nextInt(50) + 1, 0, 0, true, false, null, random) + ".com";
        String name = random.nextBoolean() ? 
                RandomStringUtils.random(random.nextInt(50), 0, 0, false, false, null, random) : null;
        
        return new MailAddress(addr, name, true);
    }

    public static void main(String[] args) throws Exception {
        
        loadAsposeLicense();

        File pstFile = null;
        File outputDir = null;
        int maxMessages = 0;
        boolean verifyMessage = false;
        long seed = 0;
        
        if(args != null && args.length > 0) {
            for(int i=0; i<args.length; i++) {
                args[i] = args[i].trim();
                if("--pstFile".equalsIgnoreCase(args[i])) {
                    pstFile = new File(args[i+1].trim());
                    i++;
                } else if("--outputDir".equalsIgnoreCase(args[i])) {
                    outputDir = new File(args[i+1].trim());
                    i++;
                } else if("--count".equalsIgnoreCase(args[i])) {
                    maxMessages = Integer.parseInt(args[i+1]);
                    i++;
                } else if("--verify".equalsIgnoreCase(args[i])) {
                    verifyMessage = true;
                } else if("--seed".equalsIgnoreCase(args[i])) {
                    seed = Long.parseLong(args[i+1]);
                } else if("--help".equalsIgnoreCase(args[i]) || "-h".equalsIgnoreCase(args[i])) {
                    printUsageThenExit();
                }
            }
        }
        
        if (pstFile == null) {
            System.out.println("Please provide path to pst file");
            printUsageThenExit();
        }
        
        if (maxMessages < 1) {
            System.out.println("Please provide number of messages");
            printUsageThenExit();
        }
        
        if (pstFile.exists()) {
            System.out.println("Deleted exist PST file: " + pstFile);
            pstFile.delete();
        }
        
        if (outputDir != null && !outputDir.exists()) {
            System.out.println("Output directory should exist if provided: " + outputDir);
            printUsageThenExit();
        }
        
        PstMissingMessageGeneratorApp app = new PstMissingMessageGeneratorApp(pstFile, maxMessages, verifyMessage, outputDir, seed);
        app.run();
        
        System.out.println("Finished, Done");
    }
    
    private static void printUsageThenExit() {
        System.out.println(
                "Parameters: --pstFile pathtopstfile --count totalmessages [--outputDir emloutputdir] [--verify] [--seed longseed] [--help]");
        System.exit(1);
    }

    public static void loadAsposeLicense() {
        try (InputStream is = PstMissingMessageGeneratorApp.class.getClassLoader()
                .getResourceAsStream("Aspose.Email.lic")) {
            License license = new License();
            license.setLicense(is);
        } catch (Exception e) {
            throw new IllegalStateException("Wrong license!");
        }
    }

}

@curtisyamada,

I have observed the issue shared by you and suggest you to please visit this documentation link for adding messages. If there is still an issue then please provide the source message file and generated PST where this message has not been saved. We will be able to investigate the issue fruiterer on our end on provision of requested information.

Investigating further the message is actually in the pst but is corrupt (You can’t do anything with the last message in the folder). The bug is only tripped when a certain set of messages are added to a folder in a certain order. If we read the same pst re-order and write to a new folder we can fix (So its not the message but some kind of meta-data) the issue but this is still a bug and forcing us to rollback to a very old version (5.8) where we know this issue does not exist.

I can’t attach a failing pst because it requires a certain number of messages which results in a large pst file. Please use the output pst from the code above.

@curtisyamada,

You may please compress the PST and host that on any file server if that is large and share the download link with us in this thread. But please provide a working sample example that we can use on our end to reproduce the issue and log it in our issue tracking system if reproduced. I also suggest you to please try using latest Aspose.Email for Java 20.4 on your end too.

Here is a corrupt pst.

I’m not sure what you mean by a working sample example, the code I attached above when run will produce a corrupt pst.

The issue still exists with 20.4 version.

@curtisyamada,

Thank you for providing the information. I have observed the issue shared by you and have created an issue with ID EMAILJAVA-34693 in our issue tracking system to further investigate and resolve the issue. This thread has been linked with the issue so that you may be notified once the issue will be addressed.

This issue still exists with 20.8 version. When can we expect a fix for this?

I ran the above code provided by @curtisyamada with following arguments:
–pstFile blah.pst --count 1000 --verify --seed 4738686600406311835

and this produces a corrupt pst file: blah.pst

@adithyab

At present the concerned issue is unresolved. We request for your patience and will share the feedback with you as soon as it will be fixed.

Do you have any estimation on this issue?
This is a big problem. and we reported exactly the same bug a few years ago.

EMAILJAVA-33521

Now the bug come back again. Could you please let us know whether they have any plan to fix it?
We cannot live with a corrupted PST file.

PST file is the only reason we use your aspose.email library.
If it’s not stable enough, then we have to look for other solution.

@curtisyamada

This issue is in progress and we are hopeful to get this issue resolved in upcoming Aspose.Email for Java 20.9. We request for your patience in this regard.