/*
 * Copyright (c) 2019-2025 XMLmind Software. All rights reserved.
 *
 * Author: Hussein Shafie
 *
 * This file is part of several XMLmind projects.
 * For conditions of distribution and use, see the accompanying legal.txt file.
 */
package com.xmlmind.util;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.io.File;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.io.Reader;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.net.URLConnection;
import java.net.URL;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

/**
 * A utility class allowing to load a text file.
 * For example, a CSS stylesheet starting with a BOM, a <code>@charset</code> 
 * or no special encoding specification.
 * <p>Unlike {@link FileUtil#loadString} and {@link URLUtil#loadString}, 
 * this utility class implements the detection of the encoding.
 * <p>Note that the detection of the encoding always succeeds
 * because it uses a fallback value.
 */
public final class LoadText {
    private LoadText() {}

    /**
     * Equivalent to {@link #loadText(URL, boolean, int, 
     * String, String[], EncodingDetector[])
     * loadText(FileUtil.fileToURL(file), true, -1, 
     * fallbackEncoding, encoding, detectors)}.
     */
    public static String loadText(File file, 
                                  String fallbackEncoding, String[] encoding, 
                                  EncodingDetector... detectors) 
        throws IOException {
        return loadText(FileUtil.fileToURL(file), true, -1,
                        fallbackEncoding, encoding, detectors);
    }

    /**
     * Equivalent to {@link #loadText(URL, boolean, int, 
     * String, String[], EncodingDetector[])
     * loadText(url, true, -1, fallbackEncoding, encoding, detectors)}.
     */
    public static String loadText(URL url, 
                                  String fallbackEncoding,  String[] encoding, 
                                  EncodingDetector... detectors) 
        throws IOException {
        return loadText(url, true, -1, fallbackEncoding, encoding, detectors);
    }

    /**
     * Loads the contents of specified text file. 
     *
     * @param url the location of the text file.
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * No effect unless <code>url</code> is an 
     * <code>http</code>/<code>https</code> URL.
     * @param timeout specifies both connect and read timeout values 
     * in milliseconds. 0 means: infinite timeout.
     * A negative value means: default value. 
     * @param fallbackEncoding the fallback encoding.
     * May be <code>null</code> in which case a sensible value 
     * (generally {@link SystemUtil#defaultEncoding}) is automatically 
     * determined.
     * @param encoding the encoding actually used to load the text 
     * is copied there. May be <code>null</code>.
     * @param detectors unless a BOM is found, specified encoding detectors are
     * used to parse the first lines of the file, possibly containing 
     * an encoding specification like <code>@charset "UTF-8";</code>.
     * @return the contents of the text file
     * @exception IOException if there is an I/O problem
     */
    public static String loadText(URL url, boolean followRedirects, int timeout,
                                  String fallbackEncoding,  String[] encoding, 
                                  EncodingDetector... detectors) 
        throws IOException {
        URLConnection connection = URLUtil.openConnectionNoCache(url);
        if (timeout >= 0) {
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        }
        connection = URLUtil.checkHttpConnection(connection, followRedirects);

        if (fallbackEncoding == null) {
            // Note that a contentType is available even for 
            // a "file://" connection.
            String contentType = connection.getContentType();
            if (contentType != null) {
                fallbackEncoding = URLUtil.contentTypeToCharset(contentType);
            }
        }

        String loaded = null;

        InputStream in = connection.getInputStream();
        try {
            loaded = loadText(in, fallbackEncoding, encoding, detectors);
        } finally {
            in.close();
        }

        return loaded;
    }

    /**
     * Loads the contents of specified text source. 
     * <p>This method implements the detection of the encoding.
     * <p>Note that the detection of the encoding always works 
     * because it uses a fallback value.
     *
     * @param in the text source.
     * @param fallbackEncoding the fallback encoding.
     * May be <code>null</code> in which case a sensible value 
     * (generally {@link SystemUtil#defaultEncoding}) is automatically 
     * determined.
     * @param encoding the encoding actually used to load the text 
     * is copied there. May be <code>null</code>.
     * @param detectors unless a BOM is found, specified encoding detectors are
     * used to parse the first lines of the file, possibly containing 
     * an encoding specification like <code>@charset "UTF-8";</code>.
     * @return the contents of the text source
     * @exception IOException if there is an I/O problem
     */
    public static String loadText(InputStream in, 
                                  String fallbackEncoding, String[] encoding,
                                  EncodingDetector... detectors) 
        throws IOException {
        Reader src = createReader(in, fallbackEncoding, encoding, detectors);
        return loadChars(src);
    }

    /**
     * Load the characters contained in specified source.
     * 
     * @param in the character source
     * @return the contents of the character source
     * @exception IOException if there is an I/O problem
     */
    public static String loadChars(Reader in) 
        throws IOException {
        StringBuilder buffer = new StringBuilder();
        char[] chars = new char[65536];
        int count;

        while ((count = in.read(chars, 0, chars.length)) != -1) {
            if (count > 0) {
                buffer.append(chars, 0, count);
            }
        }

        return buffer.toString();
    }

    /**
     * Creates a reader which can be used to read the contents 
     * of specified text source.
     *
     * @param in the text source.
     * @param fallbackEncoding the fallback encoding.
     * May be <code>null</code> in which case a sensible value 
     * (generally {@link SystemUtil#defaultEncoding}) is automatically 
     * determined.
     * @param encoding the encoding actually used to load the text 
     * is copied there. May be <code>null</code>.
     * @param detectors unless a BOM is found, specified encoding detectors are
     * used to parse the first lines of the file, possibly containing 
     * an encoding specification like <code>@charset "UTF-8";</code>.
     * @return a reader allowing to read the contents of the text source.
     * This reader will automatically skip the BOM if any.
     * @exception IOException if there is an I/O problem
     */
    public static Reader createReader(InputStream in, 
                                      String fallbackEncoding, 
                                      String[] encoding,
                                      EncodingDetector... detectors) 
        throws IOException {
        byte[] bytes = new byte[32768];
        int byteCount = -1;

        PushbackInputStream in2 = new PushbackInputStream(in, bytes.length);
        try {
            int count = in2.read(bytes, 0, bytes.length);
            if (count > 0) {
                in2.unread(bytes, 0, count);
            }
            byteCount = count;
        } catch (IOException ignored) {}

        int[] bomLength = new int[1];
        String charset = detectEncoding(bytes, byteCount, bomLength, detectors);
        if (bomLength[0] > 0) {
            // We don't want to read the BOM.
            in2.skip(bomLength[0]);
        }

        if (charset == null) {
            charset = fallbackEncoding;
            if (charset == null) {
                charset = SystemUtil.defaultEncoding();
            }
        }

        if (encoding != null) {
            encoding[0] = charset;
        }
        return new InputStreamReader(in2, charset);
    }

    // -----------------------------------------------------------------------
    // Encoding detectors
    // -----------------------------------------------------------------------

    /**
     * Detects an encoding by parsing an ASCII encoding specification 
     * (example: <code>@charset "UTF-8";</code>).
     */
    public interface EncodingDetector {
        /**
         * Parses specified few text lines and returns found encoding if any.
         *
         * @param asciiHead the first lines of a text file.
         * @return found, <em>valid</em> encoding; 
         * <code>null</code> if not found or invalid. 
         */
        public String detectEncoding(String asciiHead);
    }

    /**
     * A base class which checks for validity the encoding returned by 
     * {@link EncodingDetectorBase#doDetectEncoding}.
     */
    public static abstract class EncodingDetectorBase 
                           implements EncodingDetector {
        public String detectEncoding(String asciiHead) {
            String encoding = doDetectEncoding(asciiHead);
            return checkEncoding(encoding);
        }

        /**
         * Parses specified few text lines and returns found encoding if any.
         *
         * @param asciiHead the first lines of a text file.
         * @return found encoding; <code>null</code> if not found. 
         * <p>Returned encoding may be invalid or empty. This will be checked
         * later.
         */
        protected abstract String doDetectEncoding(String asciiHead);
    }

    /**
     * Returns the canonical name of <code>encoding</code> if valid; 
     * <code>null</code> otherwise.
     */
    public static final String checkEncoding(String encoding) {
        if (encoding != null) {
            encoding = encoding.trim();
            if (encoding.length() == 0) {
                encoding = null;
            } else {
                try {
                    Charset charset = Charset.forName(encoding);
                    encoding = charset.name();
                } catch (Exception ignored) {
                    encoding = null;
                }
            }
        }

        return encoding;
    }

    // ----------------------------------
    // XMLEncodingDetector
    // ----------------------------------

    /**
     * Detects an encoding by parsing
     * <code>&lt;?xml encoding="ENCODING"?&gt;</code>.
     */
    public static final class XMLEncodingDetector 
                        extends EncodingDetectorBase {
        private static final Pattern PATTERN = 
            Pattern.compile("encoding\\s*=\\s*['\"]([^'\"]+)['\"]");

        protected String doDetectEncoding(String asciiHead) {
            String encoding = null;

            if (asciiHead.startsWith("<?xml")) {
                Matcher matcher = PATTERN.matcher(asciiHead);
                if (matcher.find()) {
                    encoding = matcher.group(1).trim();
                }

                if (encoding == null || encoding.length() == 0) {
                    encoding = "UTF-8";
                }
            }

            return encoding;
        }
    }

    /**
     * A ready-to-use instance of {@link XMLEncodingDetector}.
     */
    public static final XMLEncodingDetector XML_ENCODING_DETECTOR = 
        new XMLEncodingDetector();

    // ----------------------------------
    // KeywordBasedDetector
    // ----------------------------------

    /**
     * Detects an encoding by parsing
     * <code>KEYWORD "ENCODING";</code>, 
     * for example <code>@charset "ENCODING";</code>.
     */
    public static final class KeywordBasedDetector 
                        extends EncodingDetectorBase {
        public final String keyword;
        private final Pattern pattern;

        public KeywordBasedDetector(String keyword) {
            assert(keyword != null && keyword.trim().length() > 0);
            this.keyword = keyword;

            pattern = Pattern.compile(Pattern.quote(keyword) + 
                                      "\\s*['\"]([^'\"]+)['\"]\\s*;");
        }

        protected String doDetectEncoding(String asciiHead) {
            String encoding = null;

            Matcher matcher = pattern.matcher(asciiHead);
            if (matcher.find()) {
                encoding = matcher.group(1).trim();
            }

            return encoding;
        }
    }

    /**
     * A ready-to-use instance of {@link KeywordBasedDetector
     * KeywordBasedDetector("&#64;charset")} (CSS stylesheets).
     */
    public static final KeywordBasedDetector CSS_CHARSET_DETECTOR = 
        new KeywordBasedDetector("@charset");

    // ----------------------------------
    // HTMLCharsetDetector
    // ----------------------------------

    /**
     * Detects an encoding by parsing
     * <code>&lt;meta charset="ENCODING" &gt;</code> or 
     * <code>&lt;meta http-equiv="Content-Type" 
     * content="text/html; charset=ENCODING"&gt;</code>.
     */
    public static final class HTMLCharsetDetector 
                        extends EncodingDetectorBase {
        private static final Pattern PATTERN = 
          Pattern.compile("<meta.+charset\\s*=\\s*['\"]?([\\w-]+)['\"]?[^>]*>", 
                          Pattern.CASE_INSENSITIVE|Pattern.DOTALL);

        protected String doDetectEncoding(String asciiHead) {
            String encoding = null;

            Matcher matcher = PATTERN.matcher(asciiHead);
            if (matcher.find()) {
                encoding = matcher.group(1).trim();
            }

            return encoding;
        }
    }

    /**
     * A ready-to-use instance of {@link HTMLCharsetDetector}.
     */
    public static final HTMLCharsetDetector HTML_CHARSET_DETECTOR = 
        new HTMLCharsetDetector();

    // ----------------------------------
    // EmacsStyleDetector
    // ----------------------------------

    /**
     * Detects an encoding by parsing
     * <code>-*- coding: ENCODING -*-</code>.
     */
    public static final class EmacsStyleDetector 
                        extends EncodingDetectorBase {
        private static final String[] SUFFIXES = { "-unix", "-dos", "-mac" };

        protected String doDetectEncoding(String asciiHead) {
            String encoding = null;

            int start = asciiHead.indexOf("-*-");
            if (start >= 0) {
                start += 3;
                int end = asciiHead.indexOf("-*-", start);
                if (end >= 0) {
                    String[] vars = 
                        StringUtil.split(asciiHead.substring(start, end), ';');
                    for (String var : vars) {
                        var = var.trim();
                        if (var.startsWith("coding:")) {
                            encoding = var.substring(7).trim();
                            
                            for (String suffix : SUFFIXES) {
                                if (encoding.endsWith(suffix)) {
                                    encoding = encoding.substring(
                                        0, encoding.length()-suffix.length());
                                    // Found suffix.
                                    break;
                                }
                            }
                            // Found encoding.
                            break;
                        }
                    }
                }
            }

            return encoding;
        }
    }

    /**
     * A ready-to-use instance of {@link EmacsStyleDetector}.
     */
    public static final EmacsStyleDetector EMACS_STYLE_DETECTOR = 
        new EmacsStyleDetector();

    /**
     * A ready-to-use array containing all {@link EncodingDetector}s.
     */
    public static final EncodingDetector[] ALL_ENCODING_DETECTORS = {
        XML_ENCODING_DETECTOR,
        HTML_CHARSET_DETECTOR,
        CSS_CHARSET_DETECTOR,
        EMACS_STYLE_DETECTOR
    };

    // -----------------------------------------------------------------------
    // detectEncoding
    // -----------------------------------------------------------------------

    /**
     * Detect encoding by examining specified bytes which 
     * have been read at the very start of a text file.
     *
     * @param bytes bytes read at the beginning of a text file.
     * @param byteCount number of bytes read at the beginning of a text file.
     * @param bomLength the length of the BOM is stored as the first element 
     * of this array. This allows to skip the BOM. May be <code>null</code>.
     * @param detectors unless a BOM is found, specified encoding detectors are
     * used to parse the first lines of the file, possibly containing 
     * an encoding specification like <code>@charset "UTF-8";</code>.
     * @return encoding if detected; <code>null</code> otherwise
     */
    @SuppressWarnings("fallthrough")
    public static String detectEncoding(byte[] bytes, int byteCount, 
                                        int[] bomLength, 
                                        EncodingDetector... detectors) {
        if (bomLength != null) {
            bomLength[0] = 0;
        }

        String encoding = null;

        Encoding guess = guessEncoding(bytes, 0, byteCount);
        //System.err.println("guessEncoding --> " + guess);
        switch (guess) {
        case UTF16_BE_BOM:
            if (bomLength != null) {
                bomLength[0] = 2;
            }
            /*FALLTHROUGH*/
        case UTF16_BE:
            encoding = "UTF-16BE";
            break;

        case UTF16_LE_BOM:
            if (bomLength != null) {
                bomLength[0] = 2;
            }
            /*FALLTHROUGH*/
        case UTF16_LE:
            encoding = "UTF-16LE";
            break;

        case UTF8_BOM:
            if (bomLength != null) {
                bomLength[0] = 3;
            }
            encoding = "UTF-8";
            break;

        case UCS4_BE_BOM:
        case UCS4_BE:
        case UCS4_LE_BOM:
        case UCS4_LE:
        case UCS4_2143_BOM:
        case UCS4_2143:
        case UCS4_3412_BOM:
        case UCS4_3412:
        case EBCDIC:
            // Not ASCII compatible. Give up.
            return null;
        }

        if (encoding == null) {
            // Unsupported characters are replaced by U+FFFD.
            String asciiHead = null;
            try {
                asciiHead = new String(bytes, 0, byteCount, "US-ASCII");
            } catch (Exception ignored) {}

            if (asciiHead != null && asciiHead.length() > 0) {
                for (EncodingDetector detector : detectors) {
                    encoding = detector.detectEncoding(asciiHead);
                    //System.err.println(detector + " --> " + encoding);
                    if (encoding != null) {
                        break;
                    }
                }
            }
        }

        return encoding;
    }

    // -----------------------------------------------------------------------
    // guessEncoding
    // -----------------------------------------------------------------------

    /**
     * The <code>UTF-16BE</code> BOM (Byte Order Mark).
     */
    public static final byte[] BOM_UTF16_BE = { (byte)0xFE, (byte)0xFF };

    /**
     * The <code>UTF-16LE</code> BOM (Byte Order Mark).
     */
    public static final byte[] BOM_UTF16_LE = { (byte)0xFF, (byte)0xFE };

    /**
     * The <code>UTF-8</code> BOM (Byte Order Mark).
     */
    public static final byte[] BOM_UTF8 = {(byte)0xEF, (byte)0xBB, (byte)0xBF};

    /**
     * Encoding returned by {@link #guessEncoding}.
     */
    public enum Encoding {
        UCS4_BE_BOM,
        UCS4_LE_BOM,
        UCS4_2143_BOM,
        UCS4_3412_BOM,
        UTF16_BE_BOM,
        UTF16_LE_BOM,
        UTF8_BOM,
        UCS4_BE,
        UCS4_LE,
        UCS4_2143,
        UCS4_3412,
        UTF16_BE,
        UTF16_LE,
        ASCII_COMPATIBLE,
        EBCDIC,
        UNKNOWN
    }

    /**
     * Guess the encoding of a text file by examining its first few bytes.
     *
     * @param bytes byte buffer.
     * @param offset byte buffer offset.
     * @param length byte buffer length. At least 4 for this function to work.
     * @return encoding if detected; <code>Encoding.UNKNOWN</code> otherwise
     */
    public static Encoding guessEncoding(byte[] bytes, int offset, int length) {
        if (length < 4) {
            return Encoding.UNKNOWN;
        }

        // '@'=0x40, 'c'=0x63|'C'=0x43, 'h'=0x68|'H'=0x48, 'a'=0x61|'A'=0x41

        int b0 = (bytes[offset] & 0xFF);
        int b1 = (bytes[offset+1] & 0xFF);
        int b2 = (bytes[offset+2] & 0xFF);
        int b3 = (bytes[offset+3] & 0xFF);

        if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF) {
            return Encoding.UCS4_BE_BOM;
        } else if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00) {
            return Encoding.UCS4_LE_BOM;
        } else if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFF && b3 == 0xFE) {
            return Encoding.UCS4_2143_BOM;
        } else if (b0 == 0xFE && b1 == 0xFF && b2 == 0x00 && b3 == 0x00) {
            return Encoding.UCS4_3412_BOM;
        } else if (b0 == 0xFE && b1 == 0xFF && (b2 != 0x00 || b3 != 0x00)) {
            return Encoding.UTF16_BE_BOM;
        } else if (b0 == 0xFF && b1 == 0xFE && (b2 != 0x00 || b3 != 0x00)) {
            return Encoding.UTF16_LE_BOM;
        } else if (b0 == 0xEF && b1 == 0xBB && b2 == 0xBF) {
            return Encoding.UTF8_BOM;
        } else if (b0 == 0x00 && b1 == 0x00 && b2 == 0x00 && b3 == 0x40) {
            return Encoding.UCS4_BE;
        } else if (b0 == 0x40 && b1 == 0x00 && b2 == 0x00 && b3 == 0x00) {
            return Encoding.UCS4_LE;
        } else if (b0 == 0x00 && b1 == 0x00 && b2 == 0x40 && b3 == 0x00) {
            return Encoding.UCS4_2143;
        } else if (b0 == 0x00 && b1 == 0x40 && b2 == 0x00 && b3 == 0x00) {
            return Encoding.UCS4_3412;
        } else if (b0 == 0x00 && b1 == 0x40 && 
                   b2 == 0x00 && (b3 == 0x63 || b3 == 0x43)) {
            return Encoding.UTF16_BE;
        } else if (b0 == 0x40 && b1 == 0x00 && 
                   (b2 == 0x63 || b2 == 0x43) && b3 == 0x00) {
            return Encoding.UTF16_LE;
        } else if (b0 == 0x40 && 
                   (b1 == 0x63 || b1 == 0x43) && 
                   (b2 == 0x68 || b2 == 0x48) && 
                   (b3 == 0x61 || b3 == 0x41)) {
            return Encoding.ASCII_COMPATIBLE;
        } else {
            return Encoding.UNKNOWN;
        }
    }

    // -----------------------------------------------------------------------

    /*TEST_LOAD_TEXT
    public static void main(String[] args) throws IOException {
        if (args.length != 1) {
            System.err.println("usage: com.xmlmind.util.LoadText text_file");
            System.exit(1);
        }

        File textFile = new File(args[0]);

        String[] encoding = new String[1];
        String text = loadText(textFile, "UTF-8", encoding,
                               XML_ENCODING_DETECTOR);

        System.out.println(textFile + ": " + Integer.toString(text.length()) + 
                           "chars, encoding=" + encoding[0]);

        System.err.println("{" + text + "}");
    }
    TEST_LOAD_TEXT*/
}
