/*
 * Copyright (c) 2017-2026 XMLmind Software. All rights reserved.
 *
 * Author: Hussein Shafie
 *
 * This file is part of several XMLmind projects.
 * For conditions of distribution and use, see the accompanying legal.txt file.
 */
package com.xmlmind.util;

import java.io.IOException;
import java.io.File;
import java.io.InputStream;
import java.io.FileOutputStream;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.net.HttpURLConnection;
import java.net.URLStreamHandler;
import java.util.Comparator;
import java.util.regex.PatternSyntaxException;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

/**
 * A collection of utility functions (static methods) operating on URLs.
 * <p>Work with any hierarchical URLs. Does not work with opaque URLs, except
 * for a few functions which work with "<code>jar:</code>" URLs.
 * <p>Note that, for these few functions, the path of "<code>jar:</code>" URL
 * (e.g. <code>jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class</code>) is
 * everything after "<code>!/</code>", including the leading "<code>/</code>".
 */
public final class URLUtil {
    private URLUtil() {}

    /**
     * A ready-to-use empty list of URLs.
     */
    public static final URL[] EMPTY_LIST = new URL[0];

    /**
     * Same as <code>new URL(spec)</code>, except that non-ASCII characters
     * and other illegal characters such as spaces possibly found in
     * <code>spec</code> are <i>%HH</i>-encoded.
     * 
     * @see URIComponent#encode
     */
    public static URL createURL(String spec)
        throws MalformedURLException {
        if (!spec.startsWith("data:")) {
            spec = URIComponent.encode(spec);
        }
        return newURL(spec);
    }

    /**
     * Same as <code>new URL(context, spec)</code>, except that non-ASCII
     * characters and other illegal characters such as spaces possibly found
     * in <code>spec</code> are <i>%HH</i>-encoded.
     * 
     * @see URIComponent#encode
     */
    public static URL createURL(URL context, String spec)
        throws MalformedURLException {
        if (!spec.startsWith("data:")) {
            spec = URIComponent.encode(spec);
        }
        return newURL(context, spec);
    }

    /**
     * Same as <code>new URL(context, spec, handler)</code>, except that
     * non-ASCII characters and other illegal characters such as spaces
     * possibly found in <code>spec</code> are <i>%HH</i>-encoded.
     * 
     * @see URIComponent#encode
     */
    public static URL createURL(URL context, String spec,
                                URLStreamHandler handler) 
        throws MalformedURLException {
        if (!spec.startsWith("data:")) {
            spec = URIComponent.encode(spec);
        }
        return newURL(context, spec, handler);
    }
    
    /**
     * A cover for {@link java.net.URL#URL(String)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(String spec)
        throws MalformedURLException {
        return new URL(spec);
    }

    /**
     * A cover for {@link java.net.URL#URL(URL, String)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(URL context, String spec)
        throws MalformedURLException {
        return new URL(context, spec);
    }

    /**
     * A cover for {@link java.net.URL#URL(URL, String, URLStreamHandler)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(URL context, String spec,
                             URLStreamHandler handler) 
        throws MalformedURLException {
        return new URL(context, spec, handler);
    }

    /**
     * A cover for {@link java.net.URL#URL(String, String, String)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(String protocol, String host, String file) 
        throws MalformedURLException {
        return new URL(protocol, host, file);
    }

    /**
     * A cover for {@link java.net.URL#URL(String, String, int, String)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(String protocol, String host, int port,
                             String file) 
        throws MalformedURLException {
        return new URL(protocol, host, port, file);
    }

    /**
     * A cover for {@link java.net.URL#URL(String, String, int, String, 
     * URLStreamHandler)}.
     */
    @SuppressWarnings("deprecation")
    public static URL newURL(String protocol, String host, int port,
                             String file, URLStreamHandler handler) 
        throws MalformedURLException {
        return new URL(protocol, host, port, file, handler);
    }

    // -----------------------------------------------------------------------

    /**
     * Returns <code>true</code> if specified URL is a <code>file:</code> URL,
     * otherwise returns <code>false</code>
     */
    public static boolean isFileURL(URL url) {
        return "file".equals(url.getProtocol());
    }

    /**
     * Returns <code>true</code> if specified URL is a <code>jar:</code> URL,
     * otherwise returns <code>false</code>
     */
    public static boolean isJarURL(URL url) {
        return "jar".equals(url.getProtocol());
    }

    /**
     * Returns <code>true</code> if specified URL is a <code>data:</code> URL,
     * otherwise returns <code>false</code>
     */
    public static boolean isDataURL(URL url) {
        return "data".equals(url.getProtocol());
    }

    /**
     * Converts a <code>file:</code> URL to a File.
     * <p>On Windows, this function converts a "file:" URL having a host
     * (other than "localhost") to an UNC filename. For example, 
     * it converts "<code>file://foo/bar/gee.txt</code>"
     * to "<code>\\foo\bar\gee.txt</code>".
     * 
     * @param url the URL to be converted
     * @return an absolute File or <code>null</code> if <code>url</code>
     * cannot be converted to a File (for example, because <code>url</code> is
     * not a <code>file:</code> URL)
     * @see #isFileURL
     * @see FileUtil#fileToURL
     * @see #urlOrFile
     */
    public static File urlToFile(URL url) {
        if (!isFileURL(url)) {
            return null;
        }

        if (SystemUtil.IS_WINDOWS) {
            String host = url.getHost();
            if (host != null && host.length() > 0 && 
                !"localhost".equalsIgnoreCase(host)) {
                // Special processing of "file://server/path" ---

                // getPath() does not include the query or the fragment.
                String location = url.getPath();
                if (location.length() == 0) {
                    location = "/";
                }
                location = "file:/C:" + location;
                     
                String filename;
                try {
                    URI uri = new URI(URIComponent.encode(location));
                    filename = (new File(uri)).getPath();
                } catch (Exception ignored) {
                    return null;
                }

                // Replace leading "C:" by "\\server".
                filename = "\\\\" + host + filename.substring(2);
                File file = new File(filename);

                try {
                    file = file.getCanonicalFile();
                } catch (IOException ignored) {}

                return file;
            } else {
                return convertURLToFile(url);
            }
        } else {
            return convertURLToFile(url);
        }
    }

    private static File convertURLToFile(URL url) {
        assert(isFileURL(url));

        try {
            // Ignore authority, query and fragment.
            String location = url.getPath();
            if (location.length() == 0) {
                location = "/";
            }
            location = "file://" + location;

            URI uri = new URI(URIComponent.encode(location));
            
            if (SystemUtil.IS_WINDOWS) {
                File file = new File(uri);

                try {
                    file = file.getCanonicalFile();
                } catch (IOException ignored) {}

                return file;
            } else {
                // Do not use getCanonicalFile on Unix because this resolves
                // symlinks.

                uri = uri.normalize();
                return new File(uri);
            }
        } catch (Exception ignored) {
            return null;
        }
    }

    /**
     * Similar to <code>java.net.URL.toURI()</code> except that this utility
     * will not throw a <code>java.net.URISyntaxException</code> if the URL
     * spec contains illegal characters such as spaces. In such case, special
     * efforts are made to nevertheless return a URI equivalent to specified
     * URL.
     * 
     * @param url URL to be converted
     * @return converted URI or <code>null</code> if this really cannot be
     * done
     */
    public static URI urlToURI(URL url) {
        String location = url.toExternalForm();
        try {
            return new URI(URIComponent.encode(location));
        } catch (URISyntaxException ignored) {
            return null;
        }
    }

    // -----------------------------------------------------------------------

    /**
     * Equivalent to {@link #urlOrFile(String, boolean, boolean, URL)
     * urlOrFile(path, false, false, null)}.
     */
    public static URL urlOrFile(String path) {
        return urlOrFile(path, /*checkAbsolute*/ false, /*allowDir*/ false, 
                         /*base*/ null);
    }

    /**
     * Equivalent to {@link #urlOrFile(String, boolean, boolean, URL)
     * urlOrFile(path, checkAbsolute, false, null)}.
     */
    public static URL urlOrFile(String path, boolean checkAbsolute) {
        return urlOrFile(path, checkAbsolute, /*allowDir*/ false, 
                         /*base*/ null);
    }

    /**
     * Equivalent to {@link #urlOrFile(String, boolean, boolean, URL)
     * urlOrFile(path, checkAbsolute, allowDir, null)}.
     */
    public static URL urlOrFile(String path, boolean checkAbsolute, 
                                boolean allowDir) {
        return urlOrFile(path, checkAbsolute, allowDir, /*base*/ null);
    }

    /**
     * Returns an URL created from specified path. First, this convenience
     * function attempts to convert specified path to an URL. If this fails,
     * specified path is considered to be the name of an <em>existing</em>
     * file or directory. If this filename conforms to specified requirements
     * (<code>checkAbsolute</code>, <code>allowDir</code>), 
     * it is converted to an URL
     * using {@link FileUtil#fileToURL}.
     * 
     * @param path external form of an URL or the filename of an existing file
     * or directory. May <em>not</em> be <code>null</code> or empty.
     * <p>If the path contains newline characters, everything after the first
     * newline character, including this character, is ignored.
     * <p>The reason for this is that Web browsers such as Firefox seems to
     * append the title of the Web page after its URL.
     * @param checkAbsolute if <code>true</code>, when <code>path</code> is a
     * filename, <code>path</code> must be absolute or this function will return
     * <code>null</code>
     * @param allowDir if <code>true</code>, 
     * when <code>path</code> is a filename,
     * <code>path</code> is allowed to be not only the path of a file 
     * but also the path of a directory
     * @param baseURL which base URL to use to resolve <code>path</code> 
     * when its a relative URL. May be <code>null</code>.
     * @return an URL or <code>null</code> if specified path cannot be
     * converted to an URL given specified requirements
     */
    public static URL urlOrFile(String path, boolean checkAbsolute, 
                                boolean allowDir, URL baseURL) {
        if (path == null) {
            return null;
        }
        
        int nl = path.indexOf('\n');
        if (nl >= 0) {
            path = path.substring(0, nl);
        }
        path = path.trim();
        if (path.length() == 0) {
            return null;
        }

        // ---
        
        URL url = null;
        try {
            url = createURL(baseURL, path);
        } catch (MalformedURLException ignored) {}

        if (url == null) {
            File file = new File(path);

            if (checkAbsolute && !file.isAbsolute()) {
                return null;
            }

            if (allowDir) {
                if (!file.exists()) {
                    return null;
                }
            } else {
                if (!file.isFile()) {
                    return null;
                }
            }

            url = FileUtil.fileToURL(file);
        }

        return url;
    }

    // -----------------------------------------------------------------------

    /**
     * Returns <code>true</code> if specified URLs have the same root.
     * 
     * @see #getRoot
     */
    public static boolean sameRoot(URL url1, URL url2) {
        return ObjectUtil.equals(getRoot(url1, true), getRoot(url2, true));
    }

    /**
     * Returns the root of specified URL.
     * <p>Example: returns "http://java.sun.com/" for
     * "http://java.sun.com/docs/index.html".
     * <p>Example: returns "jar:http://www.foo.com/bar/baz.jar!/" for
     * "jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class".
     * 
     * @param url a hierachical or "jar:" URL
     * @return root of specified URL
     */
    public static URL getRoot(URL url) {
        return getRoot(url, false);
    }

    private static URL getRoot(URL url, boolean defaultPort) {
        int port = url.getPort();
        if (port < 0 && defaultPort) {
            port = url.getDefaultPort();
        }
        
        String[] path = splitPath(url);
        if (path == null) {
            path = new String[] { null, "/" };
        }
        path[1] = "/";

        String spec = 
            URIComponent.joinQuotedComponents(url.getProtocol(),
                                              url.getUserInfo(), null,
                                              url.getHost(), port,
                                              joinPath(path), 
                                              /*query*/ null,
                                              /*fragment*/ null);

        try {
            return newURL(spec);
        } catch (MalformedURLException cannotHappen) {
            cannotHappen.printStackTrace();
            return null;
        }
    }

    /**
     * Tests whether first specified URL is an ancestor of second specified URL.
     * 
     * @param ancestorURL a hierachical or "jar:" URL
     * @param url a hierachical or "jar:" URL
     * @return <code>true</code> if <code>ancestorURL</code> is equal to 
     * or is an ancestor directory of <code>url</code>; 
     * <code>false</code> otherwise.
     */
    public static boolean isAncestorOf(URL ancestorURL, URL url) {
        if (!sameRoot(ancestorURL, url)) {
            return false;
        }

        String[] path1 = splitPath(ancestorURL);
        if (path1 == null) {
            return false;
        }

        String[] path2 = splitPath(url);
        if (path2 == null) {
            return false;
        }

        if (!path1[1].endsWith("/")) {
            path1[1] += "/";
        }

        // Assumes that paths have been canonicalized, that is, ".", "..",
        // "//", etc, have been removed.
        return path2[1].startsWith(path1[1]);
    }

    // -------------------------------
    // Support for "jar:" URLs
    // -------------------------------

    private static String[] splitPath(URL url) {
        String path = url.getPath();
        if (path.length() == 0) {
            return null;
        }

        String part1 = null;
        String part2 = null;

        if (isJarURL(url)) {
            int pos = path.indexOf("!/");
            if (pos < 0) {
                return null;
            }

            part1 = path.substring(0, pos+1); // Ends with "!".
            part2 = path.substring(pos+1); // Starts with "/".
        } else {
            part2 = path;
        }

        return new String[] { part1, part2 };
    }

    private static String joinPath(String[] parts) {
        if (parts[0] == null) {
            return parts[1];
        } else {
            return parts[0] + parts[1];
        }
    }

    /**
     * Returns the parent of specified URL, if any. Returned URL has a path
     * which ends with '/'.
     * <p>Examples:
     * <ul>
     * <li>Returns "http://java.sun.com/docs/" for
     * "http://java.sun.com/docs/index.html".
     * <li>Returns <code>null</code> for "http://java.sun.com/".
     * <li>Returns "jar:http://www.foo.com/bar/baz.jar!/COM/foo/" for 
     * "jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class".
     * </ul>
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @return parent of specified URL or <code>null</code> for root URLs.
     * @see URIComponent#getRawParentPath(String, boolean)
     */
    public static URL getParent(URL url) {
        String[] path = splitPath(url);
        if (path == null) {
            return null;
        }

        path[1] = URIComponent.getRawParentPath(path[1]);
        if (path[1] == null) {
            return null;
        }

        String spec = 
            URIComponent.joinQuotedComponents(url.getProtocol(),
                                              url.getUserInfo(), null,
                                              url.getHost(), url.getPort(),
                                              joinPath(path), 
                                              /*query*/ null,
                                              /*fragment*/ null);

        try {
            return newURL(spec);
        } catch (MalformedURLException cannotHappen) {
            cannotHappen.printStackTrace();
            return null;
        }
    }

    /**
     * Same as {@link #getRawPath} except that the returned value is
     * decoded using {@link URIComponent#decode}.
     */
    public static String getPath(URL url) {
        String path= getRawPath(url);
        return (path == null)? null : URIComponent.decode(path);
    }

    /**
     * Returns the raw (that is, possibly containing %HH escapes) path, 
     * if specified URL has a path.
     * <p>Example: returns "/index.html" for "http://www.acme.com/index.html".
     * <p>Example: returns "/COM/foo/Quux.class" for 
     * "jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class".
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @return the raw path or <code>null</code> if specified URL has no path
     * (this is not consistent with <code>URL.getPath</code> which returns the
     * empty string in such case)
     */
    public static String getRawPath(URL url) {
        String[] path = splitPath(url);
        return (path == null)? null : path[1];
    }

    /**
     * Same as {@link #getRawBaseName} except that the returned value is
     * decoded using {@link URIComponent#decode}.
     */
    public static String getBaseName(URL url) {
        String baseName= getRawBaseName(url);
        return (baseName == null)? null : URIComponent.decode(baseName);
    }

    /**
     * Returns the raw (that is, possibly containing %HH escapes) basename
     * part of the path, if specified URL has a path.
     * <p>Example: returns "index.html" for "http://www.acme.com/index.html".
     * <p>Example: returns "png" for "http://www.acme.com/icons/png/".
     * <p>Example: returns "Quux.class" for 
     * "jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class".
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @return basename or <code>null</code> if specified URL has no path
     * (this is not consistent with <code>URL.getPath</code> which returns the
     * empty string in such case)
     * @see URIComponent#getRawBaseName
     */
    public static String getRawBaseName(URL url) {
        String[] path = splitPath(url);
        return (path == null)? null : URIComponent.getRawBaseName(path[1]);
    }

    /**
     * Same as {@link #getRawExtension} except that the returned value is
     * decoded using {@link URIComponent#decode}.
     */
    public static String getExtension(URL url) {
        String extension= getRawExtension(url);
        return (extension == null)? null : URIComponent.decode(extension);
    }

    /**
     * Returns the raw (that is, possibly containing %HH escapes) extension of
     * the path, if specified URL has a path. The extension does not include a
     * leading dot '.'.
     * <p>Example: returns "html" for "http://www.acme.com/index.html".
     * <p>Example: returns <code>null</code> for 
     * "http://www.acme.com/icons/png/".
     * <p>Example: returns "class" for 
     * "jar:http://www.foo.com/bar/baz.jar!/COM/foo/Quux.class".
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @return extension or <code>null</code> if specified URL has no path
     * (this is not consistent with <code>URL.getPath</code> which returns the
     * empty string in such case)
     * @see URIComponent#getRawExtension
     */
    public static String getRawExtension(URL url) {
        String[] path = splitPath(url);
        return (path == null)? null : URIComponent.getRawExtension(path[1]);
    }

    /**
     * Same as {@link #setRawExtension} except that specified extension is
     * quoted using {@link URIComponent#quotePath}.
     */
    public static URL setExtension(URL url, String extension) {
        if (extension != null) {
            extension = URIComponent.quotePath(extension);
        }
        return setRawExtension(url, extension);
    }

    /**
     * Changes the extension of specified URL to specified extension.
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @param extension new extension. Assumed to have been quoted using
     * {@link URIComponent#quotePath}. May be <code>null</code> which means:
     * remove the extension.
     * @return an URL identical to <code>url</code> 
     * except that its extension has
     * been changed or removed.
     * <p>Returns same URL if specified URL has no path or its path ends with
     * '/'.
     * @see URIComponent#setRawExtension
     */
    public static URL setRawExtension(URL url, String extension) {
        String[] path = splitPath(url);
        if (path == null || path[1].endsWith("/")) {
            return url;
        }

        path[1] = URIComponent.setRawExtension(path[1], extension);

        String spec = 
            URIComponent.joinQuotedComponents(url.getProtocol(),
                                              url.getUserInfo(), null,
                                              url.getHost(),
                                              url.getPort(),
                                              joinPath(path),
                                              url.getQuery(),
                                              url.getRef());
        try {
            return newURL(spec);
        } catch (MalformedURLException cannotHappen) {
            cannotHappen.printStackTrace();
            return null;
        }
    }

    /**
     * Same as <code>URL.getRef</code> except that the returned value is
     * decoded using {@link URIComponent#decode}.
     */
    public static String getFragment(URL url) {
        String fragment = url.getRef();
        return (fragment == null)? null : URIComponent.decode(fragment);
    }

    /**
     * Same as {@link #setRawFragment} except that specified fragment is
     * quoted using {@link URIComponent#quoteFragment}.
     */
    public static URL setFragment(URL url, String fragment) {
        if (fragment != null) {
            fragment = URIComponent.quoteFragment(fragment);
        }
        return setRawFragment(url, fragment);
    }

    /**
     * Changes the fragment of specified URL to specified fragment.
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @param fragment new fragment. Assumed to have been quoted using {@link
     * URIComponent#quoteFragment}. May be <code>null</code> which means:
     * remove the fragment.
     * @return an URL identical to <code>url</code> 
     * except that its fragment has
     * been changed or removed. Returns <code>url</code> as is 
     * if specified URL has no path.
     * @see URIComponent#setRawFragment
     */
    public static URL setRawFragment(URL url, String fragment) {
        String[] path = splitPath(url);
        if (path == null) {
            return url;
        }

        String location = url.toExternalForm();
        location = URIComponent.setRawFragment(location, fragment);

        try {
            return newURL(location);
        } catch (MalformedURLException cannotHappen) {
            cannotHappen.printStackTrace();
            return null;
        }
    }

     /**
      * Same as {@link #getRawUserName} except that the returned value is
      * decoded using {@link URIComponent#decode}.
      */
     public static String getUserName(URL url) {
        String userName = getRawUserName(url);
        return (userName == null)? null : URIComponent.decode(userName);
     }

     /**
      * Returns the raw (that is, possibly containing <i>%HH</i> escapes) user
      * name, if a user info is found in specified URL. Returns
      * <code>null</code> otherwise.
      */
     public static String getRawUserName(URL url) {
        String userInfo = url.getUserInfo();
        if (userInfo != null) {
            int colon = userInfo.indexOf(':');
            if (colon < 0) {
                return userInfo;
            } else if (colon > 0) {
                return userInfo.substring(0, colon);
            } else {
                return null;
            }
        } else {
            return null;
        }
    }

    /**
     * Same as {@link #getRawUserPassword} except that the returned value is
     * decoded using {@link URIComponent#decode}.
     */
    public static String getUserPassword(URL url) {
        String password = getRawUserPassword(url);
        return (password == null)? null : URIComponent.decode(password);
    }

    /**
     * Returns the raw (that is, possibly containing <i>%HH</i> escapes) user
     * password, if a user info is found in specified URL. Returns
     * <code>null</code> otherwise.
     */
    public static String getRawUserPassword(URL url) {
        String userInfo = url.getUserInfo();
        if (userInfo != null) {
            int colon = userInfo.indexOf(':');
            if (colon + 1 < userInfo.length()) {
                return userInfo.substring(colon + 1);
            } else {
                return null;
            }
        } else {
            return null;
        }
    }

    /**
     * Same as {@link #setRawUserInfo} except that specified user info is
     * quoted using {@link URIComponent#quoteUserInfo}.
     */
    public static URL setUserInfo(URL url, String userName, String password) {
        if (userName != null) {
            userName = URIComponent.quoteUserInfo(userName);
        }
        if (password != null) {
            password = URIComponent.quoteUserInfo(password);
        }
        return setRawUserInfo(url, userName, password);
    }

    /**
     * Changes the user info of specified URL to specified user info.
     * 
     * @param url a hierachical or "<code>jar:</code>" URL
     * @param userName new username. Assumed to have been quoted using {@link
     * URIComponent#quoteUserInfo}. May be <code>null</code>, which means:
     * remove user info.
     * @param password new password. Assumed to have been quoted using {@link
     * URIComponent#quoteUserInfo}. May be <code>null</code>, which means:
     * password not specified.
     * @return an URL identical to <code>url</code> except 
     * that its user info has been changed or removed.
     */
    public static URL setRawUserInfo(URL url,
                                     String userName, String password) {
        if (userName == null) {
            password = null;
        }

        String spec = 
            URIComponent.joinQuotedComponents(url.getProtocol(),
                                              userName, password,
                                              url.getHost(), url.getPort(),
                                              url.getPath(), 
                                              url.getQuery(),
                                              url.getRef());

        try {
            return newURL(spec);
        } catch (MalformedURLException cannotHappen) {
            cannotHappen.printStackTrace();
            return null;
        }
    }

    /*TEST_URL_PART
    public static void main(String[] args) {
        for (int i = 0; i < args.length; ++i) {
            try {
                URL url = URLUtil.createURL(args[i]);

                System.out.print("'" + url + "'");
                System.out.print(" (protocol=" + url.getProtocol());
                System.out.print(", host=" + url.getHost());
                System.out.print(", port=" + url.getPort());
                System.out.print(", authority=" + url.getAuthority());
                System.out.print(", path=" + url.getPath());
                System.out.print(", file=" + url.getFile());
                System.out.print(", ref=" + url.getRef());
                System.out.println(", query=" +  url.getQuery() + ")");
                System.out.println("\tgetRoot='" + 
                                   getRoot(url) + "'");
                System.out.println("\tgetParent='" + 
                                   getParent(url) + "'");
                System.out.println("\tgetPath='" + 
                                   getPath(url) + "'");
                System.out.println("\tgetBaseName='" + 
                                   getBaseName(url) + "'");
                System.out.println("\tgetExtension='" + 
                                   getExtension(url) + "'");
                System.out.println("\tsetExtension='" + 
                                   setExtension(url, "foo bar") + "'");
                System.out.println("\tTrim extension='" + 
                                   setExtension(url, null) + "'");
                System.out.println("\tgetFragment='" + 
                                   getFragment(url) + "'");
                System.out.println("\tsetFragment='" + 
                                   setFragment(url, "x/y z") + "'");
                System.out.println("\tTrim fragment='" + 
                                   setFragment(url, null) + "'");
                System.out.println("\tgetUserName='" + 
                                   getUserName(url) + "'");
                System.out.println("\tgetUserPassword='" + 
                                   getUserPassword(url) + "'");
                if (!isJarURL(url)) {
                    System.out.println("\tsetUserInfo='" + 
                                       setUserInfo(url, "x@y", "change it") + 
                                       "'");
                    System.out.println("\tTrim user info='" + 
                                       setUserInfo(url, null, null) + "'");
                }
                System.out.println("----------");
            } catch (MalformedURLException ex) {
                System.err.println("*** error: malformed URL: " + ex + "'");
            }
        }
    }
    TEST_URL_PART*/

    // -----------------------------------------------------------------------

    /**
     * Returns the path of specified URL relative to specified base URL.
     * <p>More precisely returns <code>relativePath</code> such that <code>new
     * URL(base, relativePath)</code> equals <code>url</code>.
     * 
     * @param url a hierarchical or "<code>jar:</code>" URL
     * @param base another hierarchical or "<code>jar:</code>" URL
     * @return a relative path possibly followed by the query and fragment
     * components of <code>url</code> or <code>URL.toExternalForm</code> if
     * <code>url</code> or <code>base</code> have no path or
     * if <code>url</code> and
     * <code>base</code> don't have the same root
     * @see URIComponent#getRawRelativePath
     */
    public static String getRawRelativePath(URL url, URL base) {
        if (SystemUtil.IS_WINDOWS &&
            isFileURL(url) &&
            isFileURL(base)) {
            // On Windows, "file:" URLs may have different volumes (e.g.
            // file:/C:/foo and file:/D:/bar). When this is the case, do not
            // attempt to compute a relative path.

            File file = urlToFile(url);
            File baseFile = urlToFile(base);
            if (file != null && 
                baseFile != null && 
                FileUtil.getNonRelativePath(file, baseFile, null) != null) {
                return url.toExternalForm();
            }
        }

        String[] path1;
        String[] path2;
        if ((path1 = splitPath(url)) == null ||
            (path2 = splitPath(base)) == null ||
            !sameRoot(url, base)) {
            return url.toExternalForm();
        }

        String relativePath =
            URIComponent.getRawRelativePath(path1[1], path2[1]);

        return  URIComponent.joinQuotedComponents(null,
                                                  null, null,
                                                  null, -1,
                                                  relativePath, 
                                                  url.getQuery(),
                                                  url.getRef());
    }

    /*TEST_RELATIVIZE
    public static void main(String[] args) {
        int count = (args.length / 2) * 2;
        for (int i = 0; i < count; i += 2) {
            try {
                URL url = newURL(args[i]);
                URL baseURL = newURL(args[i+1]);

                String relativePath = getRawRelativePath(url, baseURL);

                System.out.println("'" + url + "'");
                System.out.println("base='" + baseURL + "'");
                System.out.println("\trelativePath='" + relativePath + "'");
                System.out.println("\tresolved='" +
                                   newURL(baseURL, relativePath) + "'");
                System.out.println("----------");
            } catch (MalformedURLException ex) {
                System.err.println("*** error: malformed URL: " + ex + "'");
            }
        }
    }
    TEST_RELATIVIZE*/

    // -----------------------------------------------------------------------

    /**
     * Same as {@link #toDisplayForm} but <code>file:</code> URLs 
     * not having a fragment are displayed as plain file names.
     */
    public static String toLabel(URL url) {
        if (url.getRef() == null) {
            File file = urlToFile(url);
            if (file != null) {
                return file.getPath();
            }
        }

        return toDisplayForm(url);
    }

    /**
     * Same as <code>java.net.URL.toExternalForm</code> except that returned
     * string may contain non-ASCII characters and that, if specified URL
     * contains a password, the characters of this password are replaced by
     * <code>'*'</code>.
     * <p>Example: returns
     * <code>ftp://jjc%40xx.com:******@ftp.xx.com/pub/My%20report.doc</code>
     * for
     * <code>ftp://jjc%40xx.com:s%25same@ftp.xx.com/pub/My%20report.doc</code>.
     * 
     * @param url a hierarchical URL possibly having a fragment 
     * and a query string
     * @return display form or <code>URL.toExternalForm</code> if specified
     * URL is opaque ("jar:" URLs are opaque).
     */
    public static String toDisplayForm(URL url) {
        URI uri = urlToURI(url);
        if (uri == null || uri.isOpaque()) {
            return url.toExternalForm();
        }
            
        String userInfo = hideUserInfo(uri);

        URI uri2;
        try {
            uri2 = new URI(uri.getScheme(),
                           userInfo,
                           uri.getHost(),
                           uri.getPort(),
                           uri.getPath(),
                           uri.getQuery(),
                           uri.getFragment());
        } catch (URISyntaxException ignored) {
            return url.toExternalForm();
        }

        return uri2.toString();
    }

    private static String hideUserInfo(URI uri) {
        // Result is not 100% correct if the usename part contains a ':'.
        String userInfo = uri.getUserInfo();
        if (userInfo != null) {
            StringBuilder buffer = new StringBuilder();

            boolean hide = false;
            int count = userInfo.length();
            for (int i = 0; i < count; ++i) {
                char c = userInfo.charAt(i);

                if (!hide) {
                    buffer.append(c);
                    if (c == ':') {
                        hide = true;
                    }
                } else {
                    buffer.append('*');
                }
            }

            userInfo = buffer.toString();
        }
        return userInfo;
    }

    /**
     * Same as {@link #toLabel} except that the returned string is made
     * shorter than specified length (when possible). This function is useful
     * to display the recently opened URLs in the <b>File</b> menu of an
     * application.
     */
    public static String toShortLabel(URL url, int maxLength) {
        File file = urlToFile(url);
        if (file != null) {
            String fileName = file.getPath();
            if (File.separatorChar != '/') {
                fileName = fileName.replace(File.separatorChar, '/');
            }

            fileName = URIComponent.truncatePath(fileName, maxLength);

            if (File.separatorChar != '/') {
                fileName = fileName.replace('/', File.separatorChar);
            }

            return fileName;
        }

        return toShortDisplayForm(url, maxLength);
    }

    /**
     * Same as {@link #toDisplayForm} except that the returned string is made
     * shorter than specified length (when possible). This function is useful
     * to display the recently opened URLs in the <b>File</b> menu of an
     * application.
     * <p>Supports "<code>jar:</code>" URLs, 
     * but returns other opaque URLs as is.
     */
    public static String toShortDisplayForm(URL url, int maxLength) {
        if (isJarURL(url)) {
            String jarLocation = url.toExternalForm();

            String archiveLocation, entryPath;
            int separ = jarLocation.indexOf("!/");
            if (separ < 0) {
                // Cannot happen.
                return jarLocation;
            }

            archiveLocation = jarLocation.substring(4, separ);
            entryPath = jarLocation.substring(separ+1); // Starts with "/".

            URL archiveURL;
            try {
                archiveURL = newURL(archiveLocation);
            } catch (MalformedURLException ignored) {
                return jarLocation;
            }

            maxLength -= 5;
            int maxLength2 = Math.min(maxLength/2, entryPath.length());
            int maxLength1 = maxLength-maxLength2;
            if (maxLength1 <= 0) {
                maxLength1 = 10;
            }
            if (maxLength2 <= 0) {
                maxLength2 = 10;
            }

            StringBuilder buffer = new StringBuilder("jar:");
            buffer.append(toShortDisplayForm(archiveURL, maxLength1));
            buffer.append('!');
            buffer.append(URIComponent.truncatePath(entryPath, maxLength2));
            return buffer.toString();
        }

        // ---

        URI uri = urlToURI(url);
        if (uri == null || uri.isOpaque()) {
            return url.toExternalForm();
        }

        String userInfo = hideUserInfo(uri);

        URI uri2;
        try {
            uri2 = new URI(uri.getScheme(),
                           userInfo,
                           uri.getHost(),
                           uri.getPort(),
                           "/",
                           null,
                           null);
        } catch (URISyntaxException ignored) {
            return url.toExternalForm();
        }

        int maxLength2 = (2*maxLength)/3;
        if (maxLength2 <= 0) {
            maxLength2 = 10;
        }
        maxLength2 = Math.max(maxLength2, maxLength-uri2.toString().length());

        String path = URIComponent.truncatePath(uri.getPath(), maxLength2);

        try {
            uri2 = new URI(uri.getScheme(),
                           userInfo,
                           uri.getHost(),
                           uri.getPort(),
                           path,
                           uri.getQuery(),
                           uri.getFragment());
        } catch (URISyntaxException ignored) {
            return url.toExternalForm();
        }

        return uri2.toString();
    }

    /*TEST_URL_LABEL
    public static void main(String[] args) throws Exception {
        java.io.PrintWriter out = 
            new java.io.PrintWriter(
                new java.io.OutputStreamWriter(System.out, "ISO-8859-1"));

        for (int i = 0; i < args.length; ++i) {
            String arg = args[i];

            System.out.println(arg);
            URL url = newURL(arg);

            System.out.println("\tto label: " + 
                               toLabel(url));
            System.out.println("\tto short label: " + 
                               toShortLabel(url, 40));
            System.out.println("\tto display form: " + 
                               toDisplayForm(url));
            System.out.println("\tto short display form: " + 
                               toShortDisplayForm(url, 40));
            System.out.println("----------");
            System.out.println();
        }
    }
    TEST_URL_LABEL*/

    // -----------------------------------------------------------------------

    /**
     * Compares two URLs by their {@link URL#toExternalForm external forms}.
     */
    public static final class URLComparator implements Comparator<URL> {
        public int compare(URL u1, URL u2) {
            return u1.toExternalForm().compareTo(u2.toExternalForm());
        }
    }

    /**
     * A ready-to-use URLComparator.
     */
    public static final URLComparator COMPARATOR = new URLComparator();

    // -----------------------------------------------------------------------

    /**
     * Equivalent to {@link #exists(URL, boolean, int) exists(url, true, -1)}.
     */
    public static boolean exists(URL url) {
        boolean exists = false;
        try {
            exists = exists(url, true, -1);
        } catch (IOException ignored) {}
        
        return exists;
    }
    
    /**
     * Tests whether specified URL corresponds to an existing resource.
     * <p>This method treats "<code>file:</code>" URLs as a special, optimized,
     * case.
     *
     * @param url the URL to be tested
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * @param timeout specifies both connect and read timeout values 
     * in milliseconds. 0 means: infinite timeout.
     * A negative value means: default value. 
     * @return <code>true</code> if specified URL corresponds to an existing
     * resource; returns <code>false</code> otherwise.
     * @exception IOException if there is an I/O problem
     */
    public static boolean exists(URL url, boolean followRedirects, int timeout)
        throws IOException {
        if (url.getRef() != null) {
            url = setRawFragment(url, null);
        }

        File file = urlToFile(url);
        if (file != null) {
            return file.exists();
        }

        // ---

        URLConnection connection = openConnectionNoCache(url);
        if (timeout >= 0) {
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        }

        boolean exists = false;
        if (connection instanceof HttpURLConnection) {
            ((HttpURLConnection) connection).setRequestMethod("HEAD");
            checkHttpConnection(connection, followRedirects);
            exists = true;
        } else {
            InputStream in = connection.getInputStream();
            try {
                in.read(); // Try to read one byte; empty file returns -1.
            } finally {
                try { in.close(); } catch (IOException ignored) {}
            }
            exists = true;
        }

        return exists;
    }

    /**
     * Equivalent to {@link #lastModified(URL, boolean, int) 
     * lastModified(url, true, -1)}.
     */
    public static long lastModified(URL url) {
        long date = -1;
        try {
            date = lastModified(url, true, -1);
        } catch (IOException ignored) {}
        
        return date;
    }
    
    /**
     * Returns the date of the resource having specified URL.
     * <p>This method treats "<code>file:</code>" URLs as a special, optimized,
     * case.
     *
     * @param url the URL to be tested
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * @param timeout specifies both connect and read timeout values 
     * in milliseconds. 0 means: infinite timeout.
     * A negative value means: default value. 
     * @return A number of milliseconds since January 1, 1970 GMT. 
     * If specified URL does not exist or if this date is unknown,
     * a number which is negative or null.
     * @exception IOException if there is an I/O problem
     */
    public static long lastModified(URL url,
                                    boolean followRedirects, int timeout)
        throws IOException {
        if (url.getRef() != null) {
            url = setRawFragment(url, null);
        }

        File file = urlToFile(url);
        if (file != null) {
            return file.lastModified();
        }

        // ---

        URLConnection connection = openConnectionNoCache(url);
        if (timeout >= 0) {
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        }
        
        long date = -1;
        if (connection instanceof HttpURLConnection) {
            ((HttpURLConnection) connection).setRequestMethod("HEAD");
            checkHttpConnection(connection, followRedirects);
        } else {
            connection.connect();
        }
        date = connection.getLastModified(); // 0 if unknown

        return date;
    }

    /**
     * Establish specified HTTP connection and check whether it returns
     * a  "200 OK". 
     * <p>No effect when specified connection is not a 
     * <code>HttpURLConnection</code> (e.g. a <code>JarURLConnection</code>).
     *
     * @param connection HTTP connection to be checked
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * @exception IOException when specified HTTP connection returned
     * a code other than "200 OK". The message of the exception contains
     * information about what happened.
     */ 
    public static URLConnection checkHttpConnection(URLConnection connection,
                                                    boolean followRedirects) 
        throws IOException {
        if (!(connection instanceof HttpURLConnection)) {
            return connection;
        }
        HttpURLConnection conn = (HttpURLConnection) connection;

        int status = HttpURLConnection.HTTP_OK;
        for (int i = 0; i < 10; ++i) {
            /*
            System.err.println("Connect #" + i + " to " + conn.getURL() +
                               " = " + ObjectUtil.instanceString(conn));
            */
            
            // Default is true. However (very common) http to https
            // redirections are not followed.
            conn.setInstanceFollowRedirects(followRedirects);
            
            // Without a "User-Agent", some connections fail with a
            // read timed out exception (e.g. https://openjdk.org/).
            // Default comes from system property http.agent.
            // Fallbacks to Java/<version>.
            conn.setRequestProperty("User-Agent", "Mozilla/5.0");
            conn.setRequestProperty("Accept-Language","en-US,en;q=0.9,*;q=0.5");
            
            conn.connect();
            status = conn.getResponseCode();

            boolean redirect = false;
            if (status != HttpURLConnection.HTTP_OK && followRedirects) {
                // Not HTTP_SEE_OTHER: the redirects don't link to the
                // requested resource itself, but to another page.
                switch (status) {
                case HttpURLConnection.HTTP_MOVED_TEMP:
                case HttpURLConnection.HTTP_MOVED_PERM:
                    redirect = true;
                    break;
                }
            }
            if (!redirect) {
                // Done.
                break;
            }
            
            String location = conn.getHeaderField("Location");
            /*
            System.err.println("Redirection #" + i + " to '" + location + "'");
            */
            String method = conn.getRequestMethod();
            String cookies = conn.getHeaderField("Set-Cookie"); // Credentials?

            URL url = null;
            try {
                url = newURL(location);
            } catch (MalformedURLException ignored) {}
            if (url == null) {
                // Should not happen. Give up.
                break;
            }
            
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod(method);
            conn.setRequestProperty("Cookie", cookies); 
        }
        
        if (status != HttpURLConnection.HTTP_OK) {
            StringBuilder msg = new StringBuilder("cannot access \"");
            msg.append(conn.getURL());
            msg.append("\": error ");
            msg.append(status);
            String explain = conn.getResponseMessage();
            if (explain != null) {
                msg.append(": ");
                msg.append(explain);
            }
            
            throw new IOException(msg.toString());
        }
        
        return conn;
    }
    
    // -----------------------------------------------------------------------

    /**
     * Equivalent to {@link #loadBytes(URL, boolean, int)
     * loadBytes(url, true, -1)}.
     */
    public static byte[] loadBytes(URL url) 
        throws IOException {
        return loadBytes(url, true, -1);
    }

    /**
     * Loads the content of an URL containing binary data.
     * 
     * @param url the URL of the binary data
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * @param timeout specifies both connect and read timeout values 
     * in milliseconds. 0 means: infinite timeout.
     * A negative value means: default value. 
     * @return the loaded bytes
     * @exception IOException if there is an I/O problem
     */
    public static byte[] loadBytes(URL url,
                                   boolean followRedirects, int timeout) 
        throws IOException {
        URLConnection connection = openConnectionNoCache(url);
        if (timeout >= 0) {
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        }
        connection = checkHttpConnection(connection, followRedirects);

        byte[] loaded = null;

        InputStream in = connection.getInputStream();
        try {
            loaded = FileUtil.loadBytes(in);
        } finally {
            in.close();
        }

        return loaded;
    }

    // -----------------------------------------------------------------------

    /**
     * Copies the contents of specified URL to 
     * specified "<code>file:</code>" URL.
     * 
     * @param srcLocation URL of the source file in string form.
     * If relative, this location is relative to the current working directory.
     * @param dstLocation  URL of the destination  file in string form.
     * If relative, this location is relative to the current working directory.
     * @exception IllegalArgumentException if <code>srcLocation</code>
     * cannot be parsed as an URL or if <code>dstLocation</code>
     * cannot be parsed as a "<code>file:</code>" URL.
     * @exception IOException if an I/O problem occurs
     */
    public static int copyFile(String srcLocation, String dstLocation)
        throws IllegalArgumentException, IOException {
        URL baseURL = FileUtil.fileToURL(new File("."));

        URL url = null;
        try {
            url = createURL(baseURL, srcLocation);
        } catch (MalformedURLException ignored) {}
        if (url == null) {
            throw new IllegalArgumentException("'" + srcLocation + 
                                               "', not an URL");
        }

        File dstFile = null;
        try {
            dstFile = urlToFile(createURL(baseURL, dstLocation));
        } catch (MalformedURLException ignored) {}
        if (dstFile == null) {
            throw new IllegalArgumentException("'" + dstLocation + 
                                               "', not a 'file:' URL");
        }

        int copied = 0;

        InputStream in = openStreamNoCache(url);
        try {
            FileOutputStream out = new FileOutputStream(dstFile);
            try {
                byte[] buffer = new byte[65536];
                int count;

                while ((count = in.read(buffer)) != -1) {
                    out.write(buffer, 0, count);
                    copied += count;
                }
                out.flush();
            } finally {
                out.close();
            }
        } finally {
            in.close();
        }

        return copied;
    }

    // -----------------------------------------------------------------------

    /**
     * Equivalent to {@link #loadString(URL, String, boolean, int) 
     * loadString(url, null, true, -1)}.
     */
    public static String loadString(URL url) 
        throws IOException {
        return loadString(url, null, true, -1);
    }

    /**
     * Equivalent to {@link #loadString(URL, String, boolean, int) 
     * loadString(url, charset, true, -1)}.
     */
    public static String loadString(URL url, String charset) 
        throws IOException {
        return loadString(url, charset, true, -1);
    }

    /**
     * Loads the content of an URL containing text.
     * 
     * @param url the URL of the text resource
     * @param charset the IANA charset of the text source if known; specifying
     * <code>null</code> means detect it using the content type obtained from
     * the connection
     * @param followRedirects if <code>true</code>, follow redirections,
     * ("301: Moved Permanently", "302: Temporary Redirect") including 
     * very common <code>http</code> to <code>https</code> ones.
     * @param timeout specifies both connect and read timeout values 
     * in milliseconds. 0 means: infinite timeout.
     * A negative value means: default value. 
     * @return the loaded String
     * @exception IOException if there is an I/O problem
     */
    public static String loadString(URL url, String charset,
                                    boolean followRedirects, int timeout) 
        throws IOException {
        URLConnection connection = openConnectionNoCache(url);
        if (timeout >= 0) {
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        }
        connection = checkHttpConnection(connection, followRedirects);

        if (charset == null) {
            // Note that a contentType is available even for 
            // a file:// connection.

            String contentType = connection.getContentType();

            /*TEST_LOAD_STRING
            System.err.println("contentType=" + contentType);
            TEST_LOAD_STRING*/

            if (contentType != null) {
                charset = contentTypeToCharset(contentType);
            }
        }

        /*TEST_LOAD_STRING
        System.err.println("charset=" + charset);
        TEST_LOAD_STRING*/

        String loaded = null;

        InputStream in = connection.getInputStream();
        try {
            loaded = FileUtil.loadString(in, charset);
        } finally {
            in.close();
        }

        return loaded;
    }

    /**
     * Returns the value of the charset parameter possibly found in specified
     * content type. For example, returns "<code>utf-8</code>", when passed 
     * "<code>text/html; charset=UTF-8</code>"
     * 
     * @param contentType a content type (AKA media type) possibly having a
     * charset parameter
     * @return value of the charset parameter (lower case) if any; 
     * <code>null</code> otherwise
     */
    public static String contentTypeToCharset(String contentType) {
        String charset = null;

        if (contentType != null) {
            contentType = contentType.toLowerCase();

            int pos = contentType.indexOf("charset=");
            if (pos >= 0 && pos+8 < contentType.length()-1) {
                charset = contentType.substring(pos+8).trim();

                int length = charset.length();
                if (length >= 2 && charset.charAt(0) == '"') {
                    charset = charset.substring(1, length-1);
                }
            }
        }

        return charset;
    }

    /**
     * Parses a content type such as 
     * "<code>text/html; charset=ISO-8859-1</code>"
     * and returns the media type (for the above example 
     * "<code>text/html</code>").
     * 
     * @param contentType the content type to be parsed
     * @return the media type (lower case) if parsing was successful;
     * <code>null</code> otherwise.
     */
    public static String contentTypeToMedia(String contentType) {
        String media = null;
        if (contentType != null) {
            contentType = contentType.toLowerCase();

            int pos = contentType.lastIndexOf(';');
            if (pos < 0) {
                media = contentType.trim();
            } else if (pos > 0) {
                media = contentType.substring(0, pos).trim();
            }
        }

        return media;
    }

    /**
     * Returns a normalized string form for specified content type.
     * <p>Example: returns <code>text/html;charset=iso-8859-1</code> for 
     * <code>text/html; charset="ISO-8859-1"</code>.
     *
     * @param contentType content type to be normalized
     * @param defaultCharset charset to add as a parameter to the content type
     * when this parameter is absent. May be <code>null</code>.
     * @return normalized string form for specified content type;
     * <code>null</code> if specified content type is malformed.
     * @see #sameContentType
     */
    public static String normalizeContentType(String contentType, 
                                              String defaultCharset) {
        String media = contentTypeToMedia(contentType);
        if (media == null) {
            return null;
        }

        String charset = contentTypeToCharset(contentType);
        if (charset == null && defaultCharset != null) {
            charset = defaultCharset.toLowerCase();
        }
        if (charset == null) {
            return media;
        }

        StringBuilder buffer = new StringBuilder(media);
        buffer.append(";charset=");
        buffer.append(charset);
        return buffer.toString();
    }

    /**
     * Tests whether specified content types are identical.
     * <p>Examples:
     * <ul>
     * <li> Returns <code>true</code> for 
     * <code>text/html; charset=ISO-8859-1</code> and 
     * <code>text/html;charset="iso-8859-1"</code>.
     * <li>Returns <code>false</code> for 
     * <code>text/html; charset=ISO-8859-1</code> and <code>text/html</code>.
     * </ul>
     *
     * @param ct1 content type to be tested
     * @param ct2 content type to be tested
     * @param defaultCharset charset to add as a parameter to a content type
     * when this parameter is absent. May be <code>null</code>.
     * @return <code>true</code> if specified content types are identical;
     * <code>false</code> otherwise
     * @see #normalizeContentType
     */
    public static boolean sameContentType(String ct1, String ct2, 
                                          String defaultCharset) {
        ct1 = normalizeContentType(ct1, defaultCharset);
        ct2 = normalizeContentType(ct2, defaultCharset);
        return ((ct1 == null && ct2 == null) ||
                (ct1 != null && ct1.equals(ct2)));
    }

    /*TEST_LOAD_STRING
    public static void main(String args[]) 
        throws MalformedURLException, IOException {
        if (args.length != 2) {
            System.err.println("usage: java com.xmlmind.util.URLUtil " +
                               " src_URL dst_file");
            System.exit(1);
        }

        String text = loadString(newURL(args[0]));
        FileUtil.saveString(text, new File(args[1]));
    }
    TEST_LOAD_STRING*/

    // -----------------------------------------------------------------------

    /**
     * Similar to <code>url.openConnection</code> except that the accessed
     * resource may not be a cached copy.
     * 
     * @param url URL for which an URLConnection must be opened
     * @exception IOException if URLConnection cannot be opened
     * @see #openStreamNoCache(URL)
     */
    public static URLConnection openConnectionNoCache(URL url) 
        throws IOException {
        URLConnection connection = url.openConnection();
        connection.setUseCaches(false);
        connection.setIfModifiedSince(0);
        return connection;
    }

    /**
     * Similar to <code>url.openStream</code> except that the accessed
     * resource may not be a cached copy.
     * 
     * @param url URL for which an input stream must be opened
     * @return opened input stream
     * @exception IOException if the input stream cannot be opened
     * @see #openConnectionNoCache(URL)
     */
    public static InputStream openStreamNoCache(URL url) 
        throws IOException {
        URLConnection connection = openConnectionNoCache(url);
        return connection.getInputStream();
    }

    /**
     * Similar to <code>url.openConnection</code> except that the accessed
     * resource may be a cached copy.
     * 
     * @param url URL for which an URLConnection must be opened
     * @exception IOException if URLConnection cannot be opened
     * @see #openStreamUseCache(URL)
     */
    public static URLConnection openConnectionUseCache(URL url) 
        throws IOException {
        URLConnection connection = url.openConnection();
        connection.setUseCaches(true); // Normally the default, so just in case.
        return connection;
    }

    /**
     * Similar to <code>url.openStream</code> except that the accessed
     * resource may be a cached copy.
     * 
     * @param url URL for which an input stream must be opened
     * @return opened input stream
     * @exception IOException if the input stream cannot be opened
     * @see #openConnectionUseCache(URL)
     */
    public static InputStream openStreamUseCache(URL url) 
        throws IOException {
        URLConnection connection = openConnectionUseCache(url);
        return connection.getInputStream();
    }
}
