/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2007-2015 Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

/*
 * GenomeManager.java
 *
 * Created on November 9, 2007, 9:12 AM
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */
package org.broad.igv.feature.genome;


import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.log4j.Logger;
import org.broad.igv.DirectoryManager;
import org.broad.igv.Globals;
import org.broad.igv.event.GenomeChangeEvent;
import org.broad.igv.event.GenomeResetEvent;
import org.broad.igv.event.IGVEventBus;
import org.broad.igv.feature.*;
import org.broad.igv.feature.genome.fasta.FastaBlockCompressedSequence;
import org.broad.igv.feature.genome.fasta.FastaDirectorySequence;
import org.broad.igv.feature.genome.fasta.FastaIndexedSequence;
import org.broad.igv.feature.genome.fasta.FastaUtils;
import org.broad.igv.prefs.Constants;
import org.broad.igv.prefs.PreferencesManager;
import org.broad.igv.track.*;
import org.broad.igv.ui.IGV;
import org.broad.igv.ui.commandbar.GenomeListManager;
import org.broad.igv.ui.panel.FrameManager;
import org.broad.igv.ui.util.MessageUtils;
import org.broad.igv.ui.util.ProgressBar;
import org.broad.igv.ui.util.ProgressMonitor;
import org.broad.igv.ui.util.UIUtilities;
import org.broad.igv.ui.util.download.Downloader;
import org.broad.igv.util.*;

import java.awt.*;
import java.io.*;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;

/**
 * @author jrobinso
 */
public class GenomeManager {

    final static String GENOME_ARCHIVE_VERSION_KEY = "version";
    final static String GENOME_ARCHIVE_PROPERTY_FILE_NAME = "property.txt";
    final static String GENOME_ARCHIVE_ID_KEY = "id";
    final static String GENOME_ARCHIVE_NAME_KEY = "name";
    final static String GENOME_ORDERED_KEY = "ordered";
    final static String GENOME_GENETRACK_NAME = "geneTrackName";
    final static String GENOME_URL_KEY = "url";
    final static String GENOME_ARCHIVE_CYTOBAND_FILE_KEY = "cytobandFile";
    final static String GENOME_ARCHIVE_GENE_FILE_KEY = "geneFile";
    final static String GENOME_ARCHIVE_SEQUENCE_FILE_LOCATION_KEY = "sequenceLocation";
    final static String COMPRESSED_SEQUENCE_PATH = "compressedSequencePath";

    /**
     * Whether the sequenceLocation has been modified from the version of the .genome
     * file on the server
     */
    public static final String GENOME_ARCHIVE_CUSTOM_SEQUENCE_LOCATION_KEY = "customSequenceLocation";
    public static final String GENOME_CHR_ALIAS_FILE_KEY = "chrAliasFile";
    public static final String SEQUENCE_MAP_FILE = "sequenceMap.txt";
    private static Logger log = Logger.getLogger(GenomeManager.class);

    private static final String ACT_USER_DEFINED_GENOME_LIST_FILE = "user-defined-genomes.txt";
    public static final String TEST_USER_DEFINED_GENOME_LIST_FILE = "test-user-defined-genomes.txt";

    public static final GenomeListItem DEFAULT_GENOME = new GenomeListItem("Human hg19", "http://s3.amazonaws.com/igv.broadinstitute.org/genomes/hg19.genome", "hg19");

    private GenomeListManager genomeListManager;

    private static GenomeManager theInstance;

    private Genome currentGenome;

    private Map<String, File> localSequenceMap;

    /**
     * Map from genomeID -> GenomeListItem
     * ID comparison will be case insensitive
     */

    public synchronized static GenomeManager getInstance() {
        if (theInstance == null) {
            theInstance = new GenomeManager();
        }
        return theInstance;
    }

    private GenomeManager() {
        genomeListManager = GenomeListManager.getInstance();
        localSequenceMap = loadSequenceMap();
    }

    public void setCurrentGenome(Genome genome) {
        if (genome != null) {
            PreferencesManager.getPreferences().setLastGenome(genome.getId());
        }
        this.currentGenome = genome;
        if (genome != null) {
            if (IGV.hasInstance()) {
                IGV.getInstance().getSession().clearHistory();
                FrameManager.getDefaultFrame().setChromosomeName(genome.getHomeChromosome(), true);
            }
            IGVEventBus.getInstance().post(new GenomeChangeEvent(genome));
        }
    }


    public void loadGenomeById(String genomeId) throws IOException {

        final Genome currentGenome = getCurrentGenome();
        if (currentGenome != null && genomeId.equals(currentGenome.getId())) {
            return; // Already loaded
        }

        if (org.broad.igv.util.ParsingUtils.pathExists(genomeId)) {
            loadGenome(genomeId, null);

        } else {

            final ProgressMonitor[] monitor = {new ProgressMonitor()};
            final ProgressBar.ProgressDialog[] progressDialog = new ProgressBar.ProgressDialog[1];
            UIUtilities.invokeAndWaitOnEventThread(() -> {
                progressDialog[0] = ProgressBar.showProgressDialog(IGV.getMainFrame(), "Loading Genome...", monitor[0], false);
            });

            try {
                GenomeListItem item = genomeListManager.getGenomeListItem(genomeId);
                loadGenome(item.getPath(), monitor[0]);
            } finally {
                UIUtilities.invokeOnEventThread(() -> {
                    progressDialog[0].setVisible(false);
                });
            }


        }
    }

    public Genome loadGenome(String genomePath, ProgressMonitor monitor) throws IOException {

        try {
            log.info("Loading genome: " + genomePath);

            Genome newGenome = null;

            if (monitor != null) {
                UIUtilities.invokeAndWaitOnEventThread(() -> monitor.fireProgress(25));
            }

            // Clear Feature DB
            FeatureDB.clearFeatures();

            String altGenomePath;
            if (genomePath.endsWith(".genome")) {
                File archiveFile = getArchiveFile(genomePath);

                if (!archiveFile.exists()) {
                    return null;    // Happens if genome download was canceled.
                }

                altGenomePath = archiveFile.getAbsolutePath();
                newGenome = loadDotGenomeFile(archiveFile);
            } else if (genomePath.endsWith(".gbk") || genomePath.endsWith(".gb")) {
                altGenomePath = genomePath;
                newGenome = loadGenbankFile(genomePath);
            } else if (genomePath.endsWith(".chrom.sizes")) {
                altGenomePath = genomePath;
                newGenome = loadChromSizes(genomePath);
            } else if (genomePath.endsWith(".json")) {
                altGenomePath = genomePath;
                newGenome = loadJsonFile(genomePath);
            } else {

                // Assume a fasta file
                altGenomePath = genomePath;
                if (genomePath.endsWith(Globals.GZIP_FILE_EXTENSION)) {

                    String gziPath = genomePath + ".gzi";
                    String faiPath = genomePath + ".fai";
                    if (!(FileUtils.resourceExists(gziPath) && FileUtils.resourceExists(faiPath))) {
                        throw new GenomeException("IGV cannot readed gzipped fasta files.");

                    }
                }
                if (!FileUtils.isRemote(genomePath)) {
                    if (!(new File(genomePath)).exists()) {
                        throw new GenomeException("Cannot locate genome: " + genomePath);
                    }
                }
                newGenome = loadFastaFile(genomePath);
            }

            // Load alias files from genome source directory, if any
            String aliasPath = FileUtils.getParent(genomePath) + "/" + newGenome.getId() + "_alias.tab";
            Collection<Collection<String>> aliases = loadChrAliases(aliasPath);
            if (aliases != null) newGenome.addChrAliases(aliases);


            // Load user-defined chr aliases, if any.  This is done last so they have priority
            aliasPath = (new File(DirectoryManager.getGenomeCacheDirectory(), newGenome.getId() + "_alias.tab")).getAbsolutePath();
            aliases = loadChrAliases(aliasPath);
            if (aliases != null) newGenome.addChrAliases(aliases);

            if (monitor != null) {
                monitor.fireProgress(25);
            }

            if (IGV.hasInstance()) IGV.getInstance().resetSession(null);


            GenomeListItem genomeListItem = new GenomeListItem(newGenome.getDisplayName(), altGenomePath, newGenome.getId());
            final Set<String> serverGenomeIDs = genomeListManager.getServerGenomeIDs();

            boolean userDefined = !serverGenomeIDs.contains(newGenome.getId());
            genomeListManager.addGenomeItem(genomeListItem, userDefined);


            setCurrentGenome(newGenome);

            if (IGV.hasInstance()) {
                FeatureTrack geneFeatureTrack = newGenome.getGeneTrack();   // Can be null
                IGV.getInstance().setGenomeTracks(geneFeatureTrack);

                List<ResourceLocator> resources = newGenome.getAnnotationResources();
                if (resources != null) {
                    IGV.getInstance().loadResources(resources);
                }
            }


            log.info("Genome loaded.  id= " + newGenome.getId());

            return currentGenome;

        } catch (SocketException e) {
            throw new RuntimeException("Server connection error", e);
        }
    }

    /**
     * Define a minimal genome from a chrom.sizes file.  It is assumed (required) that the file follow the
     * UCSC naming convention  =>  [id].chrom.sizes
     *
     * @param genomePath
     * @return
     * @throws IOException
     */
    private Genome loadChromSizes(String genomePath) throws IOException {

        int firstPeriodIdx = genomePath.indexOf('.');
        String genomeId = genomePath.substring(0, firstPeriodIdx);
        List<Chromosome> chromosomes = ChromSizesParser.parse(genomePath);
        Genome newGenome = new Genome(genomeId, chromosomes);

        // Search for chr aliases

        setCurrentGenome(newGenome);
        return newGenome;

    }

    private Genome loadGenbankFile(String genomePath) throws IOException {
        Genome newGenome;
        GenbankParser genbankParser = new GenbankParser(genomePath);
        genbankParser.readFeatures(true);

        String name = genbankParser.getLocusName();
        String chr = genbankParser.getChr();

        if (!name.equals(chr)) {
            name = name + " (" + chr + ")";
        }

        byte[] seq = genbankParser.getSequence();
        Sequence sequence = new InMemorySequence(chr, seq);
        newGenome = new Genome(chr, name, sequence, true);

        String[] aliases = genbankParser.getAliases();
        if (aliases != null) {
            List<String> aliasList = new ArrayList<String>();
            aliasList.add(chr);
            for (String a : aliases) {
                aliasList.add(a);
            }
            newGenome.addChrAliases(Arrays.<Collection<String>>asList(aliasList));
        }


        setCurrentGenome(newGenome);

        if (IGV.hasInstance() && !Globals.isHeadless()) {
            FeatureTrack geneFeatureTrack = createGeneTrack(newGenome, genbankParser.getFeatures());
            newGenome.setGeneTrack(geneFeatureTrack);
        }

        FeatureDB.addFeatures(genbankParser.getFeatures(), newGenome);

        return newGenome;
    }

    /**
     * Create a Genome from a single fasta file.
     *
     * @param genomePath
     * @return
     * @throws IOException
     */
    private Genome loadFastaFile(String genomePath) throws IOException {
        Genome newGenome;// Assume its a fasta
        String fastaPath = null;
        String fastaIndexPath = null;
        if (genomePath.endsWith(".fai")) {
            fastaPath = genomePath.substring(0, genomePath.length() - 4);
            fastaIndexPath = genomePath;
        } else {
            fastaPath = genomePath;
            fastaIndexPath = genomePath + ".fai";
        }

        if (!FileUtils.resourceExists(fastaIndexPath)) {
            //Have to make sure we have a local copy of the fasta file
            //to index it
            if (!FileUtils.isRemote(fastaPath)) {
                File archiveFile = getArchiveFile(fastaPath);
                fastaPath = archiveFile.getAbsolutePath();
                fastaIndexPath = fastaPath + ".fai";

                FastaUtils.createIndexFile(fastaPath, fastaIndexPath);
            }

        }


        GenomeListItem item = genomeListManager.buildItemFromPath(fastaPath);
        if (item == null) {
            throw new IOException(fastaPath + " does not exist, could not load genome");
        }

        FastaIndexedSequence fastaSequence = fastaPath.endsWith(".gz") ?
                new FastaBlockCompressedSequence(fastaPath) :
                new FastaIndexedSequence(fastaPath);
        Sequence sequence = new SequenceWrapper(fastaSequence);
        newGenome = new Genome(item.getId(), item.getDisplayableName(), sequence, true);
        setCurrentGenome(newGenome);
        return newGenome;
    }

    private Genome loadJsonFile(String genomePath) throws IOException {

        Genome newGenome = null;

        BufferedReader reader = ParsingUtils.openBufferedReader(genomePath);
        JsonParser parser = new JsonParser();
        JsonObject json = parser.parse(reader).getAsJsonObject();

        String id = json.get("id").getAsString();
        String name = json.get("name").getAsString();
        String fastaPath = json.get("fastaURL").getAsString();
        JsonElement indexPathObject = json.get("indexURL");
        String indexPath = indexPathObject == null ? null : indexPathObject.getAsString();

        FastaIndexedSequence sequence = fastaPath.endsWith(".gz") ?
                new FastaBlockCompressedSequence(fastaPath, indexPath) :
                new FastaIndexedSequence(fastaPath, indexPath);


        ArrayList<ResourceLocator> tracks = new ArrayList<>();
        JsonArray annotations = json.getAsJsonArray("annotations");
        if (annotations != null) {
            annotations.forEach((JsonElement jsonElement) -> {
                JsonObject obj = jsonElement.getAsJsonObject();
                String trackPath = obj.get("url").getAsString();
                JsonElement trackName = obj.get("name");
                JsonElement trackIndexPath = obj.get("indexURL");
                JsonElement indexed = obj.get("indexed");
                JsonElement aliasURL = obj.get("aliasURL");

                ResourceLocator res = new ResourceLocator(trackPath);
                if (trackName != null) res.setName(trackName.getAsString());
                if (trackIndexPath != null) res.setIndexPath(trackIndexPath.getAsString());
                if (indexed != null) res.setIndexed(indexed.getAsBoolean());
                tracks.add(res);
            });
        }


        newGenome = new Genome(id, name, sequence, true);

        newGenome.setAnnotationResources(tracks);

        // TODO -- set aliases

        return newGenome;

    }

    private Collection<Collection<String>> loadChrAliases(String path) {

        // String id = genome.getId();
        // File aliasFile = new File(DirectoryManager.getGenomeCacheDirectory(), id + "_alias.tab");
        File aliasFile = new File(path);

        if (aliasFile.exists()) {

            BufferedReader br = null;

            try {
                br = new BufferedReader(new FileReader(aliasFile));
                return loadChrAliases(br);
            } catch (IOException e) {
                log.error("Error loading chr alias table", e);
                if (!Globals.isHeadless())
                    MessageUtils.showMessage("<html>Error loading chromosome alias table.  Aliases will not be avaliable<br>" +
                            e.toString());
            } finally {
                if (br != null) {
                    try {
                        br.close();
                    } catch (IOException e) {
                        e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
                    }
                }
            }
        }
        return null;
    }


    /**
     * Create a genome from a ".genome" file.  In addition to the reference sequence .genome files can optionally
     * specify cytobands and annotations.
     */
    private Genome loadDotGenomeFile(File archiveFile) throws IOException {

        Genome newGenome;

        GenomeDescriptor genomeDescriptor = parseGenomeArchiveFile(archiveFile);

        final String id = genomeDescriptor.getId();
        final String displayName = genomeDescriptor.getName();

        boolean isFasta = genomeDescriptor.isFasta();
        String[] fastaFiles = genomeDescriptor.getFastaFileNames();

        LinkedHashMap<String, List<Cytoband>> cytobandMap = null;
        if (genomeDescriptor.hasCytobands()) {
            cytobandMap = loadCytobandFile(genomeDescriptor);
        }

        String sequencePath = localSequenceMap.containsKey(genomeDescriptor.getId()) ?
                loadSequenceMap().get(genomeDescriptor.getId()).getAbsolutePath() :
                genomeDescriptor.getSequencePath();

        // Convert legacy "local fasta" .genome file
        if (genomeDescriptor.hasCustomSequenceLocation()) {
            String localPath = genomeDescriptor.getSequencePath();
            addLocalFasta(genomeDescriptor.getId(), new File(localPath));
        }

        Sequence sequence = null;
        boolean chromosOrdered = false;
        if (sequencePath == null) {
            sequence = null;
        } else if (!isFasta) {
            sequencePath = SequenceWrapper.checkSequenceURL(sequencePath);
            IGVSequence igvSequence = new IGVSequence(sequencePath);
            if (cytobandMap != null) {
                chromosOrdered = genomeDescriptor.isChromosomesAreOrdered();
                igvSequence.generateChromosomes(cytobandMap, chromosOrdered);
            }
            sequence = new SequenceWrapper(igvSequence);
        } else if (fastaFiles != null) {
            FastaDirectorySequence fastaDirectorySequence = new FastaDirectorySequence(sequencePath, fastaFiles);
            sequence = new SequenceWrapper(fastaDirectorySequence);
        } else {

            if (sequencePath.endsWith(".gz")) {
                FastaBlockCompressedSequence fastaSequence = new FastaBlockCompressedSequence(sequencePath);
                sequence = new SequenceWrapper(fastaSequence);
            } else {
                FastaIndexedSequence fastaSequence = new FastaIndexedSequence(sequencePath);
                sequence = new SequenceWrapper(fastaSequence);
            }
            chromosOrdered = true;
        }

        newGenome = new Genome(id, displayName, sequence, chromosOrdered, genomeDescriptor);

        if (cytobandMap != null) {
            newGenome.setCytobands(cytobandMap);
        }

        Collection<Collection<String>> aliases = loadChrAliases(genomeDescriptor);
        if (aliases != null) {
            newGenome.addChrAliases(aliases);
        }

        InputStream geneStream = null;
        String geneFileName = genomeDescriptor.getGeneFileName();
        if (geneFileName != null) {
            try {
                geneStream = genomeDescriptor.getGeneStream();
                if (geneFileName.endsWith(".gbk")) {
                    GenbankParser genbankParser = new GenbankParser();
                    genbankParser.readFeatures(geneStream, false);
                    FeatureTrack geneFeatureTrack = createGeneTrack(newGenome, genbankParser.getFeatures());
                    newGenome.setGeneTrack(geneFeatureTrack);
                } else {
                    BufferedReader reader = new BufferedReader(new InputStreamReader(geneStream));
                    FeatureTrack geneFeatureTrack = createGeneTrack(newGenome, reader,
                            geneFileName, genomeDescriptor.getGeneTrackName(),
                            genomeDescriptor.getUrl());

                    newGenome.setGeneTrack(geneFeatureTrack);
                }
            } finally {
                if (geneStream != null) geneStream.close();
            }
        }

        genomeDescriptor.close();
        return newGenome;
    }

    /**
     * Returns a File of the provided genomePath. If the genomePath is a URL, it will be downloaded
     * and saved in the genome cache directory.
     *
     * @param genomePath
     * @return
     * @throws MalformedURLException
     * @throws UnsupportedEncodingException
     */
    private File getArchiveFile(String genomePath) throws MalformedURLException, UnsupportedEncodingException {

        File archiveFile;

        if (HttpUtils.isRemoteURL(genomePath.toLowerCase())) {
            // We need a local copy, as there is no http zip file reader

            URL genomeArchiveURL = new URL(genomePath);
            final String tmp = URLDecoder.decode(new URL(genomePath).getFile(), "UTF-8");
            String cachedFilename = Utilities.getFileNameFromURL(tmp);

            if (!DirectoryManager.getGenomeCacheDirectory().exists()) {
                DirectoryManager.getGenomeCacheDirectory().mkdir();
            }

            archiveFile = new File(DirectoryManager.getGenomeCacheDirectory(), cachedFilename);
            refreshCache(archiveFile, genomeArchiveURL);
        } else {
            archiveFile = new File(genomePath);
        }
        return archiveFile;
    }


    /**
     * Load the cytoband file specified in the genome descriptor.
     *
     * @param genomeDescriptor
     * @return Cytobands as a map keyed by chromosome
     */
    private LinkedHashMap<String, List<Cytoband>> loadCytobandFile(GenomeDescriptor genomeDescriptor) {
        InputStream is = null;
        try {

            is = genomeDescriptor.getCytoBandStream();
            BufferedReader reader = new BufferedReader(new InputStreamReader(is));
            return CytoBandFileParser.loadData(reader);

        } catch (IOException ex) {
            log.warn("Error loading cytoband file", ex);
            throw new RuntimeException("Error loading cytoband file" + genomeDescriptor.cytoBandFileName);
        } finally {
            try {
                if (is != null) {
                    is.close();
                }
            } catch (IOException ex) {
                log.warn("Error closing zip stream!", ex);
            }
        }
    }

    private static Collection<Collection<String>> loadChrAliases(BufferedReader br) throws IOException {
        String nextLine = "";
        Collection<Collection<String>> synonymList = new ArrayList<Collection<String>>();
        while ((nextLine = br.readLine()) != null) {
            String[] tokens = nextLine.split("\t");
            if (tokens.length > 1) {
                Collection<String> synonyms = new ArrayList<String>();
                for (String t : tokens) {
                    String syn = t.trim();
                    if (t.length() > 0) synonyms.add(syn.trim());
                }
                synonymList.add(synonyms);
            }
        }
        return synonymList;
    }

    /**
     * Load the chromosome alias file, if any, specified in the genome descriptor.
     *
     * @param genomeDescriptor
     * @return The chromosome alias table, or null if none is defined.
     */
    private Collection<Collection<String>> loadChrAliases(GenomeDescriptor genomeDescriptor) {
        InputStream aliasStream = null;
        try {
            aliasStream = genomeDescriptor.getChrAliasStream();
            if (aliasStream != null) {
                BufferedReader reader = new BufferedReader(new InputStreamReader(aliasStream));
                return loadChrAliases(reader);
            } else {
                return null;
            }
        } catch (IOException e) {
            // We don't want to bomb if the alias load fails.  Just log it and proceed.
            log.error("Error loading chromosome alias table");
            return null;
        } finally {
            try {
                if (aliasStream != null) {
                    aliasStream.close();
                }
            } catch (IOException ex) {
                log.warn("Error closing zip stream!", ex);
            }
        }
    }


    /**
     * Refresh a locally cached genome if appropriate (newer one on server, user set preference to enable it)
     * If it doesn't have a local cache, just downloaded
     * If the cached version has a custom sequence location, that is copied over to the downloaded version
     *
     * @param cachedFile
     * @param genomeArchiveURL
     * @throws IOException
     */
    public void refreshCache(File cachedFile, URL genomeArchiveURL) {
        // Look in cache first
        try {
            if (cachedFile.exists()) {

                //File sizes won't be the same if the local version has a different sequence location
                boolean remoteModfied = HttpUtils.getInstance().remoteIsNewer(cachedFile, genomeArchiveURL);

                // Force an update of cached genome if file length does not equal remote content length
                boolean forceUpdate = remoteModfied &&
                        PreferencesManager.getPreferences().getAsBoolean(Constants.AUTO_UPDATE_GENOMES);

                if (forceUpdate) {

                    log.info("Refreshing genome: " + genomeArchiveURL.toString());

                    Downloader.download(genomeArchiveURL, cachedFile, IGV.getMainFrame());

                }
            } else {
                // Copy file directly from the server to local cache.
                Frame parent = IGV.hasInstance() ? IGV.getMainFrame() : null;
                Downloader.download(genomeArchiveURL, cachedFile, parent);
            }
        } catch (Exception e) {
            MessageUtils.showErrorMessage("An error was encountered refreshing the genome cache: " + e.getMessage(), e);
        }

    }


    /**
     * Creates a genome descriptor.
     */
    public static GenomeDescriptor parseGenomeArchiveFile(File f)
            throws IOException {


        if (!f.exists()) {
            throw new FileNotFoundException("Genome file: " + f.getAbsolutePath() + " does not exist.");
        }

        GenomeDescriptor genomeDescriptor = null;
        Map<String, ZipEntry> zipEntries = new HashMap();
        ZipFile zipFile = new ZipFile(f);

        FileInputStream fileInputStream = null;
        try {
            fileInputStream = new FileInputStream(f);
            ZipInputStream zipInputStream = new ZipInputStream(fileInputStream);
            ZipEntry zipEntry = zipInputStream.getNextEntry();

            while (zipEntry != null) {
                String zipEntryName = zipEntry.getName();
                zipEntries.put(zipEntryName, zipEntry);

                if (zipEntryName.equalsIgnoreCase(GENOME_ARCHIVE_PROPERTY_FILE_NAME)) {
                    InputStream inputStream = zipFile.getInputStream(zipEntry);
                    Properties properties = new Properties();
                    properties.load(inputStream);

                    String cytobandZipEntryName = properties.getProperty(GENOME_ARCHIVE_CYTOBAND_FILE_KEY);
                    String geneFileName = properties.getProperty(GENOME_ARCHIVE_GENE_FILE_KEY);
                    String chrAliasFileName = properties.getProperty(GENOME_CHR_ALIAS_FILE_KEY);
                    String sequencePath = properties.getProperty(GENOME_ARCHIVE_SEQUENCE_FILE_LOCATION_KEY);
                    String compressedSequencePath = properties.getProperty(COMPRESSED_SEQUENCE_PATH);

                    if ((sequencePath != null) && !HttpUtils.isRemoteURL(sequencePath)) {
                        sequencePath = getFullPath(f, sequencePath);
                    }

                    if ((compressedSequencePath != null) && !HttpUtils.isRemoteURL(compressedSequencePath)) {
                        compressedSequencePath = getFullPath(f, sequencePath);
                    }

                    boolean chrNamesAltered = parseBooleanPropertySafe(properties, "filenamesAltered");
                    boolean fasta = parseBooleanPropertySafe(properties, "fasta");
                    boolean fastaDirectory = parseBooleanPropertySafe(properties, "fastaDirectory");
                    boolean chromosomesAreOrdered = parseBooleanPropertySafe(properties, GENOME_ORDERED_KEY);
                    boolean hasCustomSequenceLocation = parseBooleanPropertySafe(properties, GENOME_ARCHIVE_CUSTOM_SEQUENCE_LOCATION_KEY);


                    String fastaFileNameString = properties.getProperty("fastaFiles");
                    String url = properties.getProperty(GENOME_URL_KEY);


                    // The new descriptor
                    genomeDescriptor = new GenomeZipDescriptor(
                            properties.getProperty(GENOME_ARCHIVE_NAME_KEY),
                            properties.getProperty(GENOME_ARCHIVE_ID_KEY),
                            cytobandZipEntryName,
                            geneFileName,
                            chrAliasFileName,
                            properties.getProperty(GENOME_GENETRACK_NAME, "Gene"),
                            sequencePath,
                            hasCustomSequenceLocation,
                            compressedSequencePath,
                            zipFile,
                            zipEntries,
                            chromosomesAreOrdered,
                            fasta,
                            fastaFileNameString);

                    if (url != null) {
                        genomeDescriptor.setUrl(url);
                    }

                }
                zipEntry = zipInputStream.getNextEntry();
            }
        } finally {
            try {
                if (fileInputStream != null) {
                    fileInputStream.close();
                }
            } catch (IOException ex) {
                log.warn("Error closing imported genome zip stream!", ex);
            }
        }
        return genomeDescriptor;
    }

    private static String getFullPath(File f, String sequencePath) throws IOException {
        File sequenceFolder = null;
        sequenceFolder = new File(sequencePath);
        boolean isAbsolutePath = sequenceFolder.isAbsolute() ||
                sequencePath.startsWith("/") || sequencePath.startsWith("\\");
        if (!isAbsolutePath) {
            sequenceFolder = new File(f.getParent(), sequencePath);
        }
        sequencePath = sequenceFolder.getCanonicalPath();
        sequencePath.replace('\\', '/');
        return sequencePath;
    }

    private static boolean parseBooleanPropertySafe(Properties properties, String key) {
        String propertyString = properties.getProperty(key);
        return Boolean.parseBoolean(propertyString);
    }


    /**
     * Delete .genome files from the cache directory
     */
    public void clearGenomeCache() {

        File[] files = DirectoryManager.getGenomeCacheDirectory().listFiles();
        for (File file : files) {
            if (file.getName().toLowerCase().endsWith(Globals.GENOME_FILE_EXTENSION)) {
                file.delete();
            }
        }

    }

    /**
     * Create a genome archive (.genome) file.
     *
     * @param genomeFile
     * @param cytobandFileName  A File path to a file that contains cytoband data.
     * @param refFlatFileName   A File path to a gene file.
     * @param fastaFileName     A File path to a FASTA file, a .gz file containing a
     *                          single FASTA file, or a directory containing ONLY FASTA files.
     *                          (relative to the .genome file to be created) where the sequence data for
     *                          the new genome will be written.
     * @param genomeDisplayName The unique user-readable name of the new genome.
     * @param genomeId          The id to be assigned to the genome.
     * @param monitor           A ProgressMonitor used to track progress - null,
     *                          if no progress updating is required.
     * @return GenomeListItem
     * @throws FileNotFoundException
     */
    public GenomeListItem defineGenome(File genomeFile,
                                       String cytobandFileName,
                                       String refFlatFileName,
                                       String fastaFileName,
                                       String chrAliasFileName,
                                       String genomeDisplayName,
                                       String genomeId,
                                       javax.swing.ProgressMonitor monitor)
            throws IOException {

        File refFlatFile = null;
        File cytobandFile = null;
        File chrAliasFile = null;

        if (genomeFile != null) {
            PreferencesManager.getPreferences().setLastGenomeImportDirectory(genomeFile.getParentFile());
        }

        if ((cytobandFileName != null) && (cytobandFileName.trim().length() != 0)) {
            cytobandFile = new File(cytobandFileName);
        }

        if ((refFlatFileName != null) && (refFlatFileName.trim().length() != 0)) {
            refFlatFile = new File(refFlatFileName);
        }

        if ((chrAliasFileName != null) && (chrAliasFileName.trim().length() != 0)) {
            chrAliasFile = new File(chrAliasFileName);
        }

        if (monitor != null) monitor.setProgress(25);

        (new GenomeImporter()).createGenomeArchive(genomeFile, genomeId,
                genomeDisplayName, fastaFileName, refFlatFile, cytobandFile, chrAliasFile);

        if (monitor != null) monitor.setProgress(75);

        GenomeListItem newItem = new GenomeListItem(genomeDisplayName, genomeFile.getAbsolutePath(), genomeId);
        genomeListManager.addGenomeItem(newItem, true);

        if (monitor != null) monitor.setProgress(100);

        return newItem;

    }

    /**
     * Specific to Broad Amazon servers -- use S3 downwload rather than cloudfront
     *
     * @param path
     * @return
     */
    private String convertToS3(String path) {
        if (path.startsWith("http://igvdata") || path.startsWith("https://igvdata")) {
            return path.replaceFirst("igvdata", "igv");
        } else {
            return path;
        }
    }

    public String getGenomeId() {
        return currentGenome == null ? null : currentGenome.getId();
    }

    /**
     * IGV always has exactly 1 genome loaded at a time.
     * This returns the currently loaded genome
     *
     * @return
     * @api
     */
    public Genome getCurrentGenome() {
        return currentGenome;
    }

    /**
     * Given a directory, looks for all .genome files,
     * and outputs a list of these genomes suitable for parsing by IGV.
     * Intended to be run on server periodically.
     *
     * @param inDir    Directory in which all genome files live
     * @param rootPath The path to be prepended to file names (e.g. http://igvdata.broadinstitute.org)
     * @param outPath  Path to output file, where we will write the results
     */
    public void generateGenomeList(File inDir, String rootPath, String outPath) {
        File[] genomeFiles = inDir.listFiles(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                if (name == null) return false;
                return name.toLowerCase().endsWith(".genome");
            }
        });

        PrintWriter writer;
        try {
            writer = new PrintWriter(outPath);
        } catch (FileNotFoundException e) {
            log.error("Error opening " + outPath);
            e.printStackTrace();
            return;
        }


        GenomeDescriptor descriptor;
        for (File f : genomeFiles) {
            String curLine = "";
            try {
                descriptor = parseGenomeArchiveFile(f);
                curLine += descriptor.getName();
                curLine += "\t" + rootPath + "/" + f.getName();
                curLine += "\t" + descriptor.getId();
            } catch (IOException e) {
                log.error("Error parsing genome file. Skipping " + f.getAbsolutePath());
                log.error(e);
                continue;
            }
            writer.println(curLine);
        }

        writer.close();

    }

    /**
     * @param reader        a reader for the gene (annotation) file.
     * @param genome
     * @param geneFileName
     * @param geneTrackName
     */
    public FeatureTrack createGeneTrack(Genome genome, BufferedReader reader, String geneFileName, String geneTrackName,
                                        String annotationURL) {

        FeatureDB.clearFeatures();
        FeatureTrack geneFeatureTrack = null;

        if (reader != null) {
            FeatureParser parser;
            if (geneFileName.endsWith(".embl")) {
                parser = new EmblFeatureTableParser();
            } else if (GFFFeatureSource.isGFF(geneFileName)) {
                parser = new GFFParser();
            } else {
                parser = AbstractFeatureParser.getInstanceFor(new ResourceLocator(geneFileName), genome);
            }
            if (parser == null) {
                MessageUtils.showMessage("ERROR: Unrecognized annotation file format: " + geneFileName +
                        "<br>Annotations for genome: " + genome.getId() + " will not be loaded.");
            } else {
                List<htsjdk.tribble.Feature> genes = parser.loadFeatures(reader, genome);
                String name = geneTrackName;
                if (name == null) name = "Genes";

                String id = genome.getId() + "_genes";
                geneFeatureTrack = new FeatureTrack(id, name, new FeatureCollectionSource(genes, genome));
                geneFeatureTrack.setMinimumHeight(5);
                geneFeatureTrack.setHeight(35);
                geneFeatureTrack.setTrackType(TrackType.GENE);
                geneFeatureTrack.setColor(Color.BLUE.darker());
                TrackProperties props = parser.getTrackProperties();
                if (props != null) {
                    geneFeatureTrack.setProperties(parser.getTrackProperties());
                }
                geneFeatureTrack.setUrl(annotationURL);
            }
        }
        return geneFeatureTrack;
    }

    /**
     * Create an annotation track for the genome from a supplied list of features
     *
     * @param genome
     * @param features
     */
    public FeatureTrack createGeneTrack(Genome genome, List<htsjdk.tribble.Feature> features) {

        FeatureDB.clearFeatures();
        FeatureTrack geneFeatureTrack = null;
        String name = "Annotations";

        String id = genome.getId() + "_genes";
        geneFeatureTrack = new FeatureTrack(id, name, new FeatureCollectionSource(features, genome));
        geneFeatureTrack.setMinimumHeight(5);
        geneFeatureTrack.setHeight(35);
        //geneFeatureTrack.setRendererClass(GeneRenderer.class);
        geneFeatureTrack.setColor(Color.BLUE.darker());

        return geneFeatureTrack;
    }


    public boolean downloadGenomes(GenomeListItem item, boolean downloadSequence) {

        boolean success = false;

        try {

            File archiveFile = getArchiveFile(item.getPath());                  // Has side affect of downloading .genome file

            if (downloadSequence && item.getPath().endsWith(".genome")) {

                GenomeDescriptor genomeDescriptor = parseGenomeArchiveFile(archiveFile);
                if (genomeDescriptor.isFasta()) {
                    String fastaPath = genomeDescriptor.getSequencePath();
                    File localFile = downloadFasta(fastaPath);
                    if (localFile != null) {
                        addLocalFasta(item.getId(), localFile);
                    }
                } else {
                    MessageUtils.showMessage("Fasta file is not availble for: " + genomeDescriptor.getName());
                }
            }

            success = true;

        } catch (Exception e) {
            success = false;
            MessageUtils.showErrorMessage("Error downloading genome", e);
            log.error("Error downloading genome " + item.getDisplayableName());
        }


        if (success) {
            genomeListManager.addGenomeItem(item, false);
            IGVEventBus.getInstance().post(new GenomeResetEvent());
        }

        return success;

    }


    /**
     * Download a fasta file and associated index files.
     *
     * @throws IOException
     */
    File downloadFasta(String fastaPath) throws IOException {

        File defaultDir = DirectoryManager.getFastaCacheDirectory();
        File targetDir = defaultDir;
        //File targetDir = FileDialogUtils.chooseDirectory("Select directory for sequence", defaultDir);
        if (targetDir == null) {
            targetDir = defaultDir;
        }

        String filename = Utilities.getFileNameFromURL(fastaPath);

        File localFile = new File(targetDir, filename);
        boolean downloaded = Downloader.download(new URL(fastaPath), localFile, IGV.getMainFrame());

        if (downloaded) {
            URL indexUrl = new URL(fastaPath + ".fai");
            File localIndexFile = new File(targetDir, filename + ".fai");
            downloaded = Downloader.download(indexUrl, localIndexFile, IGV.getMainFrame());
        }

        if (downloaded) {

            if (fastaPath.endsWith(".gz")) {
                URL gziUrl = new URL(fastaPath + ".gzi");
                File localGziPath = new File(targetDir, filename + ".gzi");
                downloaded = Downloader.download(gziUrl, localGziPath, IGV.getMainFrame());
            }
        }

        return downloaded ? localFile : null;
    }


    private Map<String, File> loadSequenceMap() {

        File sequenceFile = new File(DirectoryManager.getGenomeCacheDirectory(), SEQUENCE_MAP_FILE);

        localSequenceMap = new HashMap<>();

        if (sequenceFile.exists()) {
            BufferedReader br = null;

            try {
                br = new BufferedReader(new FileReader(sequenceFile));
                String nextLine;
                while ((nextLine = br.readLine()) != null) {
                    String[] tokens = nextLine.split("\t");
                    if (tokens.length > 1) {
                        File file = new File(tokens[1]);
                        if (file.exists()) {
                            localSequenceMap.put(tokens[0], file);
                        } else {
                            log.info("Sequence file not found: " + file.getAbsolutePath());
                        }
                    }
                }
            } catch (IOException e) {
                log.error("Error loading sequence map file", e);
            } finally {
                if (br != null) try {
                    br.close();
                } catch (IOException e) {
                    log.error("Error closing sequenceMap file", e);
                }
            }
        }

        return localSequenceMap;
    }


    public File getLocalFasta(String id) {
        return localSequenceMap.get(id);
    }

    public void removeLocalFasta(String id) {
        localSequenceMap.remove(id);
        updateSequenceMapFile();
    }

    private void addLocalFasta(String id, File localFile) {
        localSequenceMap.put(id, localFile);
        updateSequenceMapFile();

    }


    private void updateSequenceMapFile() {

        PrintWriter pw = null;

        try {
            File sequenceFile = new File(DirectoryManager.getGenomeCacheDirectory(), SEQUENCE_MAP_FILE);
            pw = new PrintWriter(new BufferedWriter(new FileWriter(sequenceFile)));

            for (Map.Entry<String, File> entry : localSequenceMap.entrySet()) {
                pw.println(entry.getKey() + "\t" + entry.getValue());
            }
        } catch (IOException e) {
            log.error("Error writing sequence map", e);
        } finally {
            if (pw != null) pw.close();
        }
    }

}
