/*
    * Copyright (c) 2007, Peter Mika All rights reserved.
    * 
    * Redistribution and use in source and binary forms, with or without
    * modification, are permitted provided that the following conditions are met:
    * 
    * - Redistributions of source code must retain the above copyright notice, this
    *   list of conditions and the following disclaimer.
    * - Redistributions in binary form must reproduce the above copyright notice,
   *   this list of conditions and the following disclaimer in the documentation
   *   and/or other materials provided with the distribution. 
   * - Neither the name of the openrdf.org nor the names of its contributors may
   *   be used to endorse or promote products derived from this software without
   *   specific prior written permission.
   * 
   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   * POSSIBILITY OF SUCH DAMAGE.
   * 
   */
  package org.openrdf.elmo.scutter.servlet;
  
  import java.io.File;
  import java.io.IOException;
  import java.io.PrintWriter;
  import java.net.InetAddress;
  import java.net.URL;
  import java.util.Date;
  import java.util.Iterator;
  import java.util.List;
  import java.util.regex.Pattern;
  
  import javax.servlet.ServletConfig;
  import javax.servlet.ServletException;
  import javax.servlet.http.HttpServlet;
  import javax.servlet.http.HttpServletRequest;
  import javax.servlet.http.HttpServletResponse;
  
  import org.openrdf.elmo.scutter.RetrieverFactory;
  import org.openrdf.elmo.scutter.Scutter;
  import org.openrdf.elmo.scutter.Util;
  import org.openrdf.repository.Repository;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  
  /**
   * Web interface to the Elmo scutter. See the provided user manual for the
   * documentation of initialization and request parameters.
   * 
   * 
   * @author Peter Mika (pmika@cs.vu.nl)
   *  
   */
  
  public class ScutterServlet extends HttpServlet {
  
  	/**
  	 * 
  	 */
  	private static final long serialVersionUID = -3682817881393469043L;
  
  	private final static String CONTENT_TYPE = "text/html";
  
  	//	max. number of foaf URLs to preload from Google
  	private final static int DEFAULT_PRELOAD_SIZE = 500;
  
  	private static final int MAX_URLS = 100; //maximum number of starting points
  
  	private Thread _scutterThread = null;
  
  	private Scutter _scutter = null;
  	
  	private Repository _repository = null;
  
  	private File _queueFile = null; // file used for persistent storage of the
  								   // queue
      
      private File _blacklistFile = null; // file used for persistent storage of the
         // blacklist
  
  	private int _preloadSize = DEFAULT_PRELOAD_SIZE;
  
  	private String _running = "false"; //we cannot synchronize on primitive
  									   // objects
      
  	private String[] _startURLs = null; 
  
  	public void init(ServletConfig config) throws ServletException {
  		super.init(config);
  
 		
 		_repository = Util.initRepository(config);
 		String focused = config.getInitParameter("focused");
         RetrieverFactory factory = null;
 		try {
 			if (focused != null && focused.length() > 0) {
 				factory = new RetrieverFactory(_repository, true);
 			} else {
 				factory = new RetrieverFactory(_repository, false);
 			}
 		} catch (Exception e) {
 			throw new ServletException("Factory failed to initialize", e);
 		}
         
         try {
             _scutter = new Scutter(factory);
         } catch (Exception e) {
             throw new ServletException("Scutter failed to initialize", e);
         }
         
 		//QUEUE
 		String queueName = config.getInitParameter("queue");
 		if (queueName != null) {
             _queueFile = new File(queueName); 
             _scutter.setQueueFile(_queueFile);
 		}
 		
 		//BLACKLIST
         String blacklistName = config.getInitParameter("blacklist");
         if (blacklistName != null) {
             _blacklistFile = new File(blacklistName);
             _scutter.setBlacklistFile(_blacklistFile);
         }
         
 		//PRELOADSIZE
 		String preloadSize = config.getInitParameter("preloadSize");
 		if (preloadSize != null) {
 			try {
 				_preloadSize = Integer.parseInt(preloadSize);
 			} catch (Exception e) {
 				throw new ServletException(
 						"Invalid value for <preloadSize> initialization parameter.");
 			}
 		}
 
 		//DOMAIN
 		try {
 			String domain = config.getInitParameter("domain"); //optional
 															   // argument
 			if (domain != null && !domain.equals("")) {
 				Pattern domainPattern = Pattern.compile(domain);
 				_scutter.setDomainPattern(domainPattern);
 			}
 		} catch (Exception e) {
 			throw new ServletException(
 					"Invalid value for <domain> initialization parameter.");
 		}
   
 		//VOCAB
         try {
             String vocab = config.getInitParameter("vocab"); //optional
                                                                // argument
             if (vocab != null && vocab.equalsIgnoreCase("foaf")) {               
                 factory.setFoafOnly(true);
             }
         } catch (Exception e) {
             throw new ServletException(
                     "Invalid value for <vocab> initialization parameter.");
         }
         
 		//START
 		String startString = config.getInitParameter("start"); //optional argument
 		try {
 		    _startURLs = new String[MAX_URLS];
 			if (startString != null && !startString.equals("")) {
 				String[] terms = startString.split("\\s");
 				int count = 0;
 				for (int i=0; i < Math.min(terms.length, MAX_URLS); i++) {
 					terms[i] = terms[i].trim();
 					if (terms[i].startsWith("http:")) {
 						_startURLs[count++]=terms[i];
 					}
 				}
 				_scutter.initQueue(_startURLs);
 			}
 		} catch (Exception e) {
 			throw new ServletException(e);
 		}
 
 		//METADATA
 		String metadata = config.getInitParameter("metadata"); //optional
 															   // argument
 		if (metadata != null && metadata.equalsIgnoreCase("true")) {
 			_scutter.setStoreMetadata(true);
 		}
 		
 		//SIZELIMIT
 		String sizelimit = config.getInitParameter("sizelimit"); //optional argument
 		if (sizelimit != null) {
 			try {
 				_scutter.setSizeLimit(Integer.parseInt(sizelimit));
 			} catch (NumberFormatException nfe) {
 				throw new ServletException(
 				"Invalid value for <sizelimit> initialization parameter.");
 			}
 		}
         
 		//AUTOBLACKLIST
         String autoblacklist = config.getInitParameter("autoblacklist"); //optional argument
         if (autoblacklist != null && autoblacklist.equalsIgnoreCase("false")) {
             _scutter.setAutoBlackList(false);
         } else {
             _scutter.setAutoBlackList(true);
         }
 
         //MAXTHREADS
         String maxThreadsString = config.getInitParameter("maxThreads");
         if (maxThreadsString != null) {
             try {
                 _scutter.setMaxThreads(Integer.parseInt(maxThreadsString));
             } catch (NumberFormatException nfe) {
                 throw new ServletException(
                 "Invalid value for <maxThreads> initialization parameter.");
             }
         }
         
 	}
 
 	public void doGet(HttpServletRequest request, HttpServletResponse response)
 			throws ServletException, IOException {
 
 		response.setContentType(CONTENT_TYPE);
 
 		PrintWriter out = response.getWriter();
 		out.println("<html>");
 		out.println("<head>");
 		out.println("<title>");
 		out.println("ScutterServlet");
 		out.println("</title>");
 		out.println("<link rel=\"stylesheet\" href=\"elmo.css\" type=\"text/css\">");
 		out.println("</head>");
 		out.println("<body>");
 		out.println("<br/><b>ScutterServlet on </b>"
 				+ this.getServletContext().getServerInfo() + "<b> at </b>"
 				+ InetAddress.getLocalHost() + "<b> reporting...</b>");
 		//TODO: write out repository ID
 		out.println("<br><b>Using Sesame repository</b> " + _repository);
 		out.println("<br/>");
 		String operation = request.getParameter("operation");
 		if (operation != null) {
 			if (operation.equals("start")) {
 				if (!_running.equals("true")) {
 					synchronized (_running) {
 						
 						_scutterThread = new Thread(_scutter);
 						_scutterThread.start();
 						_running = "true";
 						out
 						.println("<br/><b>Scutter started successfully.</b><br/>");
 					}
 				}
 			} else if (operation.equals("stop")) {
 				if (!_running.equals("false")) {
 					synchronized (_running) {
 						
 						try {
 						    _scutter.stop();
 						}
 						catch (IOException ioe) {
 						    out.println("Saving queue file failed, check servlet configuration.");
 						}
 						_running = "false";
 					}
 				}
 				if (!_scutterThread.isAlive()) {
 					out
 							.println("<br/><b>Scutter stopped successfully.</b><br/>");
 				}
 			} 
 			//TODO: consider removing this operation
 //			else if (operation.equals("search")) {
 //				List urlList = org.openrdf.elmo.util.Util.googleScrapeURLs(
 //						"filetype:rdf foaf:Person", _preloadSize);
 //				String[] urlArray = new String[urlList.size()];
 //				for (int i = 0; i < urlList.size(); i++) {
 //					urlArray[i] = (String) urlList.get(i);
 //				}
 //				_scutter.initQueue(urlArray);
 //				out.println("<br/><b>Scutter queue preloaded with <b>"
 //						+ urlList.size()
 //						+ "</b> URLs (may not be unique).</b><br/>");
 //			} 
 			else if (operation.equals("preloadQueue")) {
 				int loaded = 0;
 				if (_queueFile.exists()) {
 					loaded = _scutter.loadQueue();
 				}
 				out.println("<br/><b>Scutter queue preloaded from file <i>"
 						+ _queueFile + "</i> with <b>" + loaded
 						+ "</b> URLs.</b>");
 			} else if (operation.equals("clear")) {
 				if (_scutter.clear()) {
 					out
 							.println("<br/><b>Scutter queue and visited list cleared.</b><br/>");
 				} else {
 					out
 							.println("<br/><b>Error while clearing scutter repository, see logs.</b><br/>");
 				}
 				//Put back the startURL
 				if (_startURLs != null) {					
 					_scutter.initQueue(_startURLs);
 				}
 			} else if (operation.equals("add")) {
 				if (request.getParameter("url") != null
 						&& !request.getParameter("url").equals("")) {
 					try {
 						_scutter.addURL(new URL((String) request
 								.getParameter("url")));
 					} catch (Exception e) {
 						out
 								.println("<br/><b>Error while adding URL. </b><br/>");
 						e.printStackTrace();
 					}
 				} else {
 					out.println("<b/r><b>Error while adding URL.</b><br/>");
 				}
 			}
 
 		}
 		if (_scutterThread != null && _scutterThread.isAlive()) {
 			out.println("<br/>Status:" + "running");
 		} else {
 			out.println("<br/>Status:" + "stopped");
 		}
 
 		List queue = _scutter.getQueue();
 		out.println("<br/>(1) Queue size:" + queue.size());
 		out.println("<br/>(2) Seen so far:" + _scutter.getVisited().size());
 		out.println("<br/>Crawled so far (2-1):" + (_scutter.getVisited().size() - queue.size()));
 		
 
 		if (_scutter.getDomainPattern() != null) {
 			out.println("<br/>Whitelist pattern: " +_scutter.getDomainPattern().pattern());
 		}
 		out.println("<br/>");
 		out.println("<br/>Queue contents:<br/>");
 		int counter = 0;
 		synchronized (queue) {
 			Iterator it = queue.iterator();
 			while (it.hasNext()) {
 				URL nextURL = (URL) it.next();
 				out.println("<br/>" + counter++ + ". " + nextURL);
 			}
 		}
 		out.println("<br/><small>" + new Date() + "</small>");
 		out.println("</body>");
 		out.println("</html>");
 
 		out.close();
 	}
 
      /** For debugging purposes only.
      * 
      * @param args
      * @throws Exception
      */
     public static void main(String[] args) throws Exception {
         
         Repository repository = Util.initRepository(args[0], args[1]);
         
         String focused = "true";
         RetrieverFactory factory = null;
         try {
             if (focused != null && focused.length() > 0) {
                 factory = new RetrieverFactory(repository, true);
             } else {
                 factory = new RetrieverFactory(repository, false);
             }
         } catch (Exception e) {
             throw new ServletException("Factory failed to initialize", e);
         }
         Logger logger = LoggerFactory.getLogger(ScutterServlet.class);
         
         //TODO: configure Log4j logging
 //        logger.setLevel(Level.DEBUG);
 //        Logger.getLogger("org.apache").setLevel(Level.INFO);
 //        Logger.getLogger("httpclient").setLevel(Level.INFO);
         
         
         try {
             Scutter scutter = new Scutter(factory); 
             scutter.addURL(new URL(args[2]));
             scutter.run();           
             System.out.println("<br/><b>Scutter started successfully.</b><br/>");
         } catch (Exception e) {
             throw new ServletException("Scutter failed to initialize", e);
         }
         
         
         
     }
 }