package de.ugoe.cs.eventbench.web;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import de.ugoe.cs.eventbench.web.data.WebEvent;
import de.ugoe.cs.util.FileTools;
import de.ugoe.cs.util.console.Console;

/**
 * <p>
 * Provides functionality to parse log files with web request.
 * </p>
 * 
 * @author Steffen Herbold
 * @version 1.0
 */
public class WeblogParser {

	/**
	 * <p>
	 * Timeout between two sessions in milliseconds.
	 * </p>
	 */
	private long timeout;

	/**
	 * <p>
	 * Minimal length of a session. All shorter sessions will be pruned.<br>
	 * Default: 2
	 * </p>
	 */
	private int minLength = 2;

	/**
	 * <p>
	 * Maximal length of a session. All longer sessions will be pruned.<br>
	 * Default: 100
	 * </p>
	 */
	private int maxLength = 100;

	/**
	 * <p>
	 * URL of the server that generated the log that is currently parser; null
	 * of URL is not available.<br>
	 * Default: null
	 * </p>
	 */
	private String url = null;

	/**
	 * <p>
	 * Collection of generated sequences.
	 * </p>
	 */
	private List<List<WebEvent>> sequences;

	/**
	 * <p>
	 * List that stores the users (identified through their cookie id) to each
	 * sequence.
	 * </p>
	 */
	private List<String> users;

	/**
	 * <p>
	 * List that stores the frequent users (identified through their cookie id)
	 * to each sequence.
	 * </p>
	 */
	private List<String> frequentUsers;

	/**
	 * <p>
	 * Sequences for all frequent users.
	 * </p>
	 */
	private List<Collection<List<WebEvent>>> sequencesFrequentUsers;

	/**
	 * <p>
	 * Threshold that defines how many sessions of a user are require to deem
	 * the user frequent. Note, that only sessions whose lengths is in range if
	 * {@link #minLength} and {@link #maxLength} are counted.
	 * </p>
	 */
	private int frequentUsersThreshold = -1;

	/**
	 * <p>
	 * Name and path of the robot filter.
	 * </p>
	 */
	private static final String ROBOTFILTERFILE = "misc/robotfilter.txt";

	/**
	 * <p>
	 * Field that contains a regular expression that matches all robots
	 * contained in {@link #ROBOTFILTERFILE}.
	 * </p>
	 */
	private String robotRegex = null;

	/**
	 * <p>
	 * Constructor. Creates a new WeblogParser with a default timeout of
	 * 3,600,000 milliseconds (1 hour).
	 * </p>
	 */
	public WeblogParser() {
		this(3600000);
	}

	/**
	 * <p>
	 * Constructor. Creates a new WeblogParser.
	 * </p>
	 * 
	 * @param timeout
	 *            session timeout
	 */
	public WeblogParser(long timeout) {
		this.timeout = timeout;
	}

	/**
	 * <p>
	 * Returns the generated event sequences.
	 * </p>
	 * 
	 * @return generated event sequences
	 */
	public Collection<List<WebEvent>> getSequences() {
		return sequences;
	}

	/**
	 * <p>
	 * Sets the session timeout.
	 * </p>
	 * 
	 * @param timeout
	 *            new session timeout
	 */
	public void setTimeout(long timeout) {
		this.timeout = timeout;
	}

	/**
	 * <p>
	 * Sets the minimal length of a session. All sessions that contain less
	 * events will be pruned.
	 * </p>
	 * 
	 * @param minLength
	 *            new minimal length
	 */
	public void setMinLength(int minLength) {
		this.minLength = minLength;
	}

	/**
	 * <p>
	 * Sets the maximal length of a session. All sessions that contain more
	 * events will be pruned.
	 * </p>
	 * 
	 * @param maxLength
	 *            new maximal length
	 */
	public void setMaxLength(int maxLength) {
		this.maxLength = maxLength;
	}

	/**
	 * <p>
	 * Sets the URL of the server from which this log was generated. Often
	 * required for replay generation
	 * </p>
	 * 
	 * @param url
	 *            URL of the server
	 */
	public void setUrl(String url) {
		this.url = url;
	}

	/**
	 * <p>
	 * Sets the threshold for frequent users.
	 * </p>
	 * 
	 * @param threshold
	 *            threshold value; if the value is &lt;1, the sessions of the
	 *            frequent users will not be determined
	 */
	public void setFrequentUserThreshold(int threshold) {
		this.frequentUsersThreshold = threshold;
	}

	/**
	 * <p>
	 * Returns the IDs of all frequent users.
	 * </p>
	 * 
	 * @return IDs of the frequent users
	 */
	public List<String> getFrequentUsers() {
		return frequentUsers;
	}

	/**
	 * <p>
	 * Returns the sequences of all frequent users.
	 * </p>
	 * </p>
	 * 
	 * @return list of the sequences of all frequent users
	 */
	public List<Collection<List<WebEvent>>> getFrequentUserSequences() {
		return sequencesFrequentUsers;
	}

	/**
	 * <p>
	 * Parses a web log file.
	 * </p>
	 * 
	 * @param filename
	 *            name and path of the log file
	 * @throws IOException
	 *             thrown if there is a problem with reading the log file
	 * @throws FileNotFoundException
	 *             thrown if the log file is not found
	 * @throws ParseException
	 *             thrown the date format is invalid
	 */
	public void parseFile(String filename) throws IOException,
			FileNotFoundException, ParseException {
		String[] lines = FileTools.getLinesFromFile(filename);

		Map<String, List<Integer>> cookieSessionMap = new HashMap<String, List<Integer>>();
		int lastId = -1;

		SimpleDateFormat dateFormat = new SimpleDateFormat(
				"yyyy-MM-dd HH:mm:ss");
		loadRobotRegex();

		sequences = new ArrayList<List<WebEvent>>();
		users = new ArrayList<String>();

		int lineCounter = 0;
		for (String line : lines) {
			lineCounter++;
			String[] values = line.substring(1, line.length() - 1).split(
					"\" \"");

			// use cookie as session identifier
			int cookieStart = values[0].lastIndexOf('.');
			String cookie = values[0].substring(cookieStart + 1);
			String dateString = values[1];
			long timestamp = dateFormat.parse(dateString).getTime();
			String uriString = values[2];
			// String ref = values[3]; // referer is not yet used!
			String agent;
			if (values.length > 4) {
				agent = values[4];
			} else {
				agent = "noagent";
			}

			List<String> postedVars = new ArrayList<String>();
			if (values.length == 6) { // post vars found
				for (String postVar : values[5].trim().split(" ")) {
					// TODO manual filtering of bad variables, should be
					// automated
					if (!postVar.contains("and")) {
						postedVars.add(postVar);
					}
				}
			}
			if (!isRobot(agent)) {
				try {
					URI uri = new URI(uriString);
					String path = uri.getPath();
					List<String> getVars = extractGetVarsFromUri(uri);

					WebEvent event = new WebEvent(url, path, timestamp,
							postedVars, getVars);

					// find session and add event
					List<Integer> sessionIds = cookieSessionMap.get(cookie);
					if (sessionIds == null) {
						sessionIds = new ArrayList<Integer>();
						// start new session
						sessionIds.add(++lastId);
						cookieSessionMap.put(cookie, sessionIds);
						sequences.add(new LinkedList<WebEvent>());
						users.add(cookie);
					}
					Integer lastSessionIndex = sessionIds
							.get(sessionIds.size() - 1);
					List<WebEvent> lastSession = sequences
							.get(lastSessionIndex);
					long lastEventTime = timestamp;
					if (!lastSession.isEmpty()) {
						lastEventTime = lastSession.get(lastSession.size() - 1)
								.getTimestamp();
					}
					if (timestamp - lastEventTime > timeout) {
						sessionIds.add(++lastId);
						List<WebEvent> newSession = new LinkedList<WebEvent>();
						newSession.add(event);
						sequences.add(newSession);
						users.add(cookie);
					} else {
						lastSession.add(event);
					}
				} catch (URISyntaxException e) {
					Console.traceln("Ignored line " + lineCounter + ": "
							+ e.getMessage());
				}
			}
		}
		Console.traceln("" + sequences.size() + " user sequences found");
		pruneSequences();
		Console.traceln("" + sequences.size()
				+ " remaining after pruning of sequences shorter than "
				+ minLength);
		Set<String> uniqueUsers = new HashSet<String>(users);
		Console.traceln("" + uniqueUsers.size() + " unique users");
		if (frequentUsersThreshold > 0) {
			generateFrequentUserSequences(uniqueUsers);
		}
	}

	/**
	 * <p>
	 * Generates the frequent user sequences, according to the threshold
	 * {@link #frequentUsersThreshold}.
	 * </p>
	 * 
	 * @param uniqueUsers
	 *            set with all user IDs
	 */
	private void generateFrequentUserSequences(Set<String> uniqueUsers) {
		frequentUsers = new ArrayList<String>();
		sequencesFrequentUsers = new ArrayList<Collection<List<WebEvent>>>();
		for (String user : uniqueUsers) {
			List<String> tmp = new ArrayList<String>();
			tmp.add(user);
			List<String> usersCopy = new LinkedList<String>(users);
			usersCopy.retainAll(tmp);
			int size = usersCopy.size();
			if (size >= frequentUsersThreshold) {
				frequentUsers.add(user);
				Collection<List<WebEvent>> sequencesUser = new ArrayList<List<WebEvent>>();
				for (int i = 0; i < sequences.size(); i++) {
					if (users.get(i).equals(user)) {
						sequencesUser.add(sequences.get(i));
					}
				}
				sequencesFrequentUsers.add(sequencesUser);

			}
		}
		Console.traceln("" + frequentUsers.size() + " users with more than "
				+ frequentUsersThreshold + " sequences");
	}

	/**
	 * <p>
	 * Prunes sequences shorter than {@link #minLength} and longer than
	 * {@link #maxLength}.
	 * </p>
	 */
	private void pruneSequences() {
		int i = 0;
		while (i < sequences.size()) {
			if ((sequences.get(i).size() < minLength)
					|| sequences.get(i).size() > maxLength) {
				sequences.remove(i);
				users.remove(i);
			} else {
				i++;
			}
		}

	}

	/**
	 * <p>
	 * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
	 * matches all the robots defined in the file. The regular expression is
	 * stored in the field {@link #robotRegex}.
	 * </p>
	 * 
	 * @throws IOException
	 *             thrown if there is a problem reading the robot filter
	 * @throws FileNotFoundException
	 *             thrown if the robot filter is not found
	 */
	private void loadRobotRegex() throws IOException, FileNotFoundException {
		String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
		StringBuilder regex = new StringBuilder();
		for (int i = 0; i < lines.length; i++) {
			regex.append("(.*" + lines[i] + ".*)");
			if (i != lines.length - 1) {
				regex.append('|');
			}
		}
		robotRegex = regex.toString();
	}

	/**
	 * <p>
	 * Checks whether an agent is a robot.
	 * </p>
	 * 
	 * @param agent
	 *            agent that is checked
	 * @return true, if the agent is a robot; false otherwise
	 */
	private boolean isRobot(String agent) {
		return agent.matches(robotRegex);
	}

	/**
	 * <p>
	 * Parses the URI and extracts the GET variables that have been passed.
	 * </p>
	 * 
	 * @param uri
	 *            URI that is parsed
	 * @return a list with all GET variables
	 */
	private List<String> extractGetVarsFromUri(URI uri) {
		List<String> getVars = new ArrayList<String>();
		String query = uri.getQuery();
		if (query != null) {
			String[] paramPairs = query.split("&");
			for (String paramPair : paramPairs) {
				String[] paramSplit = paramPair.split("=");
				// TODO manual filtering of bad variables, should be automated
				if (!paramSplit[0].contains("and")) {
					getVars.add(paramSplit[0]);
				}
			}
		}
		return getVars;
	}
}