/* ====================================================================
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2000-2003 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Apache" and "Apache Software Foundation" and
 *     "Apache Jetspeed" must not be used to endorse or promote products
 *    derived from this software without prior written permission. For
 *    written permission, please contact apache@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache" or
 *    "Apache Jetspeed", nor may "Apache" appear in their name, without
 *    prior written permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

package org.apache.jetspeed.services.urlmanager;

//standard Java stuff
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Hashtable;
import java.util.Vector;

//turbine stuff
import org.apache.jetspeed.services.resources.JetspeedResources;

//jetspeed stuff
import org.apache.jetspeed.cache.disk.DiskCacheEntry;
import org.apache.jetspeed.cache.disk.DiskCacheUtils;
import org.apache.jetspeed.cache.disk.JetspeedDiskCache;
import org.apache.jetspeed.services.logging.JetspeedLogFactoryService;
import org.apache.jetspeed.services.logging.JetspeedLogger;

/**
<p>
Handles fetching URLs and if for some reason anything happens add it to the
BadURLManager.  There are also some util methods for downloading URLs that don't
use the Disk Cache.
</p>



@author <a href="mailto:burton@apache.org">Kevin A. Burton</a>
@author <a href="mailto:sgala@hisitech.com">Santiago Gala</a>
@version $Id: URLFetcher.java,v 1.13 2003/07/23 19:50:26 morciuch Exp $
*/
public class URLFetcher 
{
    /**
     * Static initialization of the logger for this class
     */    
    private static final JetspeedLogger logger = JetspeedLogFactoryService.getLogger(URLFetcher.class.getName());
    
    /**
    URLs that Jetspeed is currently trying to fetch in real time.
    */
    private static Hashtable realtime_urls = new Hashtable();

    /**
     *
     */
    static final boolean shouldFetchNow = 
        JetspeedResources.getBoolean( JetspeedResources.CACHE_REQUIRE_CACHED_KEY );
    
    static {
        //Looking for redirected channels...
        java.net.HttpURLConnection.setFollowRedirects(true);
    }

    public static final Reader fetch( String url ) throws IOException {
        return fetch ( url, false );
    }

    
    /**
    Try and fetch a URL as and get the content as a String and possibly add
    the URL to the BadURLManager if anything goes wrong.
    
    @param url The URL to fetch
    @param force if set to true then do not use force this entry to be in the cache...
                 IE do not use CACHE_REQUIRE_CACHED
    */
    public static final Reader fetch( String url,
                                           boolean force ) throws IOException {

        if ( ! URLManager.isOK( url ) ) {
            throw new URLNotAvailableException( url );
        }

        //SGP
        if( force == false && DiskCacheUtils.isCached( url ) == true) 
        {
            logger.info( "The url " + 
                      url + 
                      " is fetched from the Cache" );
            return JetspeedDiskCache.getInstance().getEntry( url ).getReader();
        }
        
        //do cache required checking
        if ( shouldFetchNow && 
             DiskCacheUtils.isCached( url ) == false && 
             isRealtimeURL( url ) == false &&
             force == false ) {

            logger.info( "The url " + 
                      url + 
                      " is not in the cache and will be fetched now because you have configured -> " + 
                      JetspeedResources.CACHE_REQUIRE_CACHED_KEY );
                 
            //it is possible that two thread request the same URL.
            //The refresh call in JetspeedDiskCache takes care of this.
            JetspeedDiskCache.getInstance().refresh( url );
            
            //thow an Exception that this isn't in the cache.
            throw new ContentNotAvailableException( url );
        }

        if( isRealtimeURL( url ) == true ) {
            addRealtimeURL( url );
          synchronized(url.intern())
          {
             try
             {
               //We wait for other thread to load
                url.intern().wait();
             } catch (InterruptedException e)
             {
               logger.info("Wait Interrupted");
             } finally
             {
                  removeRealtimeURL( url );
               }
          }
            // We try again
          return URLFetcher.fetch( url, force );
        } else {
            addRealtimeURL( url );
        }
        try {
            
            URL content;

	    // Determine the URL's protocol
            String protocol = url.substring(0, url.indexOf(":/"));
	    
	    // Check if a proxy is set. If no port is set, use the default port (-1)
            String proxyHost = URLManager.getProxyHost( protocol );
            if (proxyHost != null)
            {
                // Open the URL using a proxy
                content = new URL(protocol,
                                  proxyHost,
                                  URLManager.getProxyPort( protocol ),
                                  url);
            }
            else
            {
                content = new URL( url );
            }

            URLConnection conn = content.openConnection();
            return getReader( conn );
            
        } catch ( Throwable t ) {
            
            String reason = "";
            
            if ( t instanceof MalformedURLException ) {
                reason = "The URL is Malformed.";
            } else {
                reason = t.toString();
            }
            
            //if the URL couldn't be fetched because it is remote AND
            //it is not in the cache, add it to the bad URL list.
            if ( DiskCacheUtils.isCached( url ) == false ) {
                //Reported up there...
                //logger.error( t );
                URLManager.register( url, URLManagerService.STATUS_BAD, reason );
            } else {
            //it is in the cache, remove it (could be broken in cache).
            //next time we could be luckier.
                JetspeedDiskCache.getInstance().remove(url);
            }


            throw new URLNotAvailableException( reason, url );

        } finally {
            removeRealtimeURL( url );
        }

    }


    /**
    Try and fetch a URL if the copy in the cache has expired and add
    the URL to the BadURLManager if anything goes wrong.
    
    @param url The URL to fetch
    @param force if set to true then do not use force this entry to be in the cache...
                 IE do not use CACHE_REQUIRE_CACHED
    */
    public static final boolean refresh( String url) throws IOException {
        
        if ( ! URLManager.isOK( url ) ) {
            if( DiskCacheUtils.isCached(url) ) 
                JetspeedDiskCache.getInstance().remove(url);
            throw new URLNotAvailableException( url );
        }
        
        if(isRealtimeURL(url)) {
            return false;
        }

            
         DiskCacheEntry dce = null;
         if( DiskCacheUtils.isCached(url) ) {
             try {
                 dce = JetspeedDiskCache.getInstance().getEntry( url );
                 if(!dce.hasExpired())
                 {
                         return false;
                 }
                 addRealtimeURL( url );

                 //only update this if the URL on which it is based is newer 
                 //than the one on disk.
                 URL sock;
                   
                 // Determine the URL's protocol
                 String protocol = url.substring(0, url.indexOf(":/"));
           
                 // Check if a proxy is set. If no port is set, use the default port (-1)
                 String proxyHost = URLManager.getProxyHost( protocol );
                 if (proxyHost != null)
                 {
                     // Open the URL using a proxy
                     sock = new URL(protocol,
                                    proxyHost,
                                    URLManager.getProxyPort( protocol ),
                                    url);
                 }
                 else
                 {	
                     sock = new URL( url );
                 }

                 URLConnection conn = null;
                 conn = sock.openConnection();

                 File file = dce.getFile();
                 long mod = dce.getLastModified();
                 long filesize = 0;
                 if(file != null)
                 {
                     filesize = file.length();
                 }

                 if(mod > 0 || filesize > 0)
                     conn.setIfModifiedSince(mod);
                   
                 conn.connect();
                 long last = conn.getLastModified();
                 long expires = conn.getExpiration();
                 int clength = conn.getContentLength();
                 int respCode = 200;
                 if(conn instanceof HttpURLConnection) {
                     respCode = ( ( HttpURLConnection )conn ).getResponseCode();
                 }
                   
                 if (respCode != 304 /*NOT MODIFIED*/ && 
                     (clength == -1 || clength > 0) && 
                     (  last == 0 || 
                       last > dce.getLastModified()) ) {

                     logger.info( "URLFetcher: Found updated URL: " + 
                               url +
                               " Modified " + last + " Expires: " + expires +
                               " CLength: " + clength ); 
                
                     //force this URL to update.

                     JetspeedDiskCache.getInstance().getEntry( url, getReader( conn ) );
                     //Trying to deal with a problem under FreeBSD
                     conn.getInputStream().close();

                     //Set the last modified and expiration times for entry
                     //FIXME: 0 is used in FileWatcher to mean not initialized...
                     if(last > 0)
                         dce.setLastModified(last);    
                     else
                         dce.setLastModified( System.currentTimeMillis() );
                     dce.setExpirationTime(expires);


                     //removeRealtimeURL( url ); (done in finally)
                     return true;
                     //now make sure that the entry that depends on this HREF
                     //is updated in the PortletFactory.
                 } else {

                     if(last > 0)
                         dce.setLastModified(last);    
                     else
                         dce.setLastModified( System.currentTimeMillis() );
                     dce.setExpirationTime(expires);
                           
                       
                     logger.info( "DiskCacheDaemon: URL still valid: " + url +
                               " Modified " + last + " Expires: " + expires +
                               " CLength: " + clength); 
                     //removeRealtimeURL( url ); (done in finally)
                     return false;
                 }
             } catch (Throwable e) {
                 //Add as a Bad URL
                 logger.error("Throwable",  e);
                 URLManager.register( url,
                                      URLManagerService.STATUS_BAD,
                                      e.toString() );
             } finally {
                 removeRealtimeURL( url );
             }
                   
         } else {
             logger.info( "URLFetcher: Cache miss during validation! Forcing url: " + url ); 
             removeRealtimeURL( url );
             JetspeedDiskCache.getInstance().getEntry( url, true );
             return true;
         }
           return false;
                    
    }


    /**
     *
     * Return a Reader for a given HTTP connection.
     * If the connection first line contains a XML declaration
     * with encoding, honor this encoding.
     * If not, use the encoding from the HTTP connection,
     * taking ISO-8859-1 as default.
     *
    */
    static final Reader getReader( URLConnection conn )
        throws IOException, UnsupportedEncodingException {
        String enc = conn.getContentEncoding();
        if( enc == null ) {
            enc = "ISO-8859-1";
        }
        // Some XML files come with a encoding attribute inside,
        // different than the HTTP encoding. We will have
        // to start reading the Reader, read the attribute and rewind 
        // the stream, generating a new reader with the "true" encoding
        BufferedInputStream is = new BufferedInputStream( conn.getInputStream() );
        //If document is XML, find the encoding and give it priority over
        //the one returned by the connection

        //we mark for resetting later. We need a big number to ensure
        // stack of streams don't read it to fill buffers.
        is.mark( 20480 );
        BufferedReader asciiReader = new BufferedReader( new InputStreamReader( is, "ASCII" ) );
        String decl = asciiReader.readLine();
        //System.err.println( "Line: " + decl );
        String key = "encoding=\"";
        //decl nul means that the connection got reset...
        if( decl != null ) {
            int off = decl.indexOf( key );
            if( off > 0 ) {
                enc = decl.substring( off + key.length(), 
                                      decl.indexOf( '"' , off + key.length()) );
            }
        }
        logger.info("URLFetcher: found URL with encoding -> " + enc );
        //Reset the bytes read
        is.reset();
        Reader rdr = new InputStreamReader( is,
                                            enc );
        return rdr;
    }


    
    /**
    Add a URL that is downloading in realtime
    */
    static final void addRealtimeURL( String url ) {
        synchronized( realtime_urls )
        {
            Vector threads = (Vector) realtime_urls.get( url);
            if(threads != null)
               {
                if(!threads.contains(Thread.currentThread()))
                   {
                     threads.addElement(Thread.currentThread() );
                   }
               } else {
                threads = new Vector();
                threads.addElement(Thread.currentThread());
                realtime_urls.put( url, threads  );
               }
        }
        
    }
    
    /**
    Remove a URL because it isn't downloading anymore.
    */
    static final void removeRealtimeURL( String url ) {
        synchronized( realtime_urls )
        {
           Vector threads = (Vector) realtime_urls.get( url);
           if(threads != null)
               synchronized( threads  )
                   {
                    Thread realLoader = (Thread) threads.firstElement();
                    if(realLoader == Thread.currentThread())
                    {
                      synchronized(url.intern())
                     {
                      realtime_urls.remove(url);
                      url.intern().notifyAll();
                      }          
                     } else {
                     threads.removeElement(Thread.currentThread());
                     }
                    }
        }
        
    }

    /**
    Return true if this URL isn't downloading in realtime.
    */
    static final boolean isRealtimeURL( String url ) {

        synchronized( realtime_urls ) {
            return realtime_urls.get( url ) != null;
        }
            
    }

    /**
    Return the list of realtime URLs for debug
    */
    public static final Hashtable getRealtimeURLs() {
        synchronized(realtime_urls) {
            return realtime_urls;
        }
    }
    
}
