Changeset 232 for trunk

Timestamp:

10/05/11 00:39:28 (13 years ago)

Author:

sherbold

Message:

• extended de.ugoe.cs.eventbench.web.WeblogParser? with functionality to determine frequent users
• adapted command loadSessionsFromClickstream allow optional determination of frequent users

Location:

trunk/EventBenchConsole/src/de/ugoe/cs/eventbench/web

Files:

• WeblogParser.java (modified) (10 diffs)
• commands/CMDloadSessionsFromClickstream.java (modified) (5 diffs)

trunk/EventBenchConsole/src/de/ugoe/cs/eventbench/web/WeblogParser.java

@@ -10 +10 @@
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import de.ugoe.cs.eventbench.web.data.WebEvent;
@@ -69 +71 @@
         /**
          * <p>
+         * List that stores the users (identified through their cookie id) to each
+         * sequence.
+         * </p>
+         */
+        private List<String> users;
+
+        /**
+         * <p>
+         * List that stores the frequent users (identified through their cookie id)
+         * to each sequence.
+         * </p>
+         */
+        private List<String> frequentUsers;
+
+        /**
+         * <p>
+         * Sequences for all frequent users.
+         * </p>
+         */
+        private List<Collection<List<WebEvent>>> sequencesFrequentUsers;
+
+        /**
+         * <p>
+         * Threshold that defines how many sessions of a user are require to deem
+         * the user frequent. Note, that only sessions whose lengths is in range if
+         * {@link #minLength} and {@link #maxLength} are counted.
+         * </p>
+         */
+        private int frequentUsersThreshold = -1;
+
+        /**
+         * <p>
          * Name and path of the robot filter.
          * </p>
@@ -164 +198 @@
         public void setUrl(String url) {
                 this.url = url;
+        }
+
+        /**
+         * <p>
+         * Sets the threshold for frequent users.
+         * </p>
+         * 
+         * @param threshold
+         *            threshold value; if the value is &lt;1, the sessions of the
+         *            frequent users will not be determined
+         */
+        public void setFrequentUserThreshold(int threshold) {
+                this.frequentUsersThreshold = threshold;
+        }
+
+        /**
+         * <p>
+         * Returns the IDs of all frequent users.
+         * </p>
+         * 
+         * @return IDs of the frequent users
+         */
+        public List<String> getFrequentUsers() {
+                return frequentUsers;
+        }
+
+        /**
+         * <p>
+         * Returns the sequences of all frequent users.
+         * </p>
+         * </p>
+         * 
+         * @return list of the sequences of all frequent users
+         */
+        public List<Collection<List<WebEvent>>> getFrequentUserSequences() {
+                return sequencesFrequentUsers;
         }
 
@@ -192 +262 @@
 
                 sequences = new ArrayList<List<WebEvent>>();
+                users = new ArrayList<String>();
 
                 int lineCounter = 0;
@@ -216 +287 @@
                         if (values.length == 6) { // post vars found
                                 for (String postVar : values[5].trim().split(" ")) {
-                                        postedVars.add(postVar);
+                                        // TODO manual filtering of bad variables, should be
+                                        // automated
+                                        if (!postVar.contains("and")) {
+                                                postedVars.add(postVar);
+                                        }
                                 }
                         }
@@ -236 +311 @@
                                                 cookieSessionMap.put(cookie, sessionIds);
                                                 sequences.add(new LinkedList<WebEvent>());
+                                                users.add(cookie);
                                         }
                                         Integer lastSessionIndex = sessionIds
@@ -251 +327 @@
                                                 newSession.add(event);
                                                 sequences.add(newSession);
+                                                users.add(cookie);
                                         } else {
                                                 lastSession.add(event);
@@ -260 +337 @@
                         }
                 }
+                Console.traceln("" + sequences.size() + " user sequences found");
                 pruneSequences();
-        }
-
-        /**
-         * <p>
-         * Prunes sequences shorter than {@link #minLength}.
+                Console.traceln("" + sequences.size()
+                                + " remaining after pruning of sequences shorter than "
+                                + minLength);
+                Set<String> uniqueUsers = new HashSet<String>(users);
+                Console.traceln("" + uniqueUsers.size() + " unique users");
+                if (frequentUsersThreshold > 0) {
+                        generateFrequentUserSequences(uniqueUsers);
+                }
+        }
+
+        /**
+         * <p>
+         * Generates the frequent user sequences, according to the threshold
+         * {@link #frequentUsersThreshold}.
+         * </p>
+         * 
+         * @param uniqueUsers
+         *            set with all user IDs
+         */
+        private void generateFrequentUserSequences(Set<String> uniqueUsers) {
+                frequentUsers = new ArrayList<String>();
+                sequencesFrequentUsers = new ArrayList<Collection<List<WebEvent>>>();
+                for (String user : uniqueUsers) {
+                        List<String> tmp = new ArrayList<String>();
+                        tmp.add(user);
+                        List<String> usersCopy = new LinkedList<String>(users);
+                        usersCopy.retainAll(tmp);
+                        int size = usersCopy.size();
+                        if (size >= frequentUsersThreshold) {
+                                frequentUsers.add(user);
+                                Collection<List<WebEvent>> sequencesUser = new ArrayList<List<WebEvent>>();
+                                for (int i = 0; i < sequences.size(); i++) {
+                                        if (users.get(i).equals(user)) {
+                                                sequencesUser.add(sequences.get(i));
+                                        }
+                                }
+                                sequencesFrequentUsers.add(sequencesUser);
+
+                        }
+                }
+                Console.traceln("" + frequentUsers.size() + " users with more than "
+                                + frequentUsersThreshold + " sequences");
+        }
+
+        /**
+         * <p>
+         * Prunes sequences shorter than {@link #minLength} and longer than
+         * {@link #maxLength}.
          * </p>
          */
         private void pruneSequences() {
-                Console.traceln("" + sequences.size() + " user sequences found");
-                // prune sequences shorter than min-length and longer than maxLength
                 int i = 0;
                 while (i < sequences.size()) {
@@ -276 +395 @@
                                         || sequences.get(i).size() > maxLength) {
                                 sequences.remove(i);
+                                users.remove(i);
                         } else {
                                 i++;
                         }
                 }
-                Console.traceln("" + sequences.size()
-                                + " remaining after pruning of sequences shorter than "
-                                + minLength);
+
         }
 
@@ -338 +456 @@
                         for (String paramPair : paramPairs) {
                                 String[] paramSplit = paramPair.split("=");
-                                getVars.add(paramSplit[0]);
+                                // TODO manual filtering of bad variables, should be automated
+                                if (!paramSplit[0].contains("and")) {
+                                        getVars.add(paramSplit[0]);
+                                }
                         }
                 }

trunk/EventBenchConsole/src/de/ugoe/cs/eventbench/web/commands/CMDloadSessionsFromClickstream.java

@@ -5 +5 @@
 import java.security.InvalidParameterException;
 import java.text.ParseException;
+import java.util.Collection;
 import java.util.List;
 
 import de.ugoe.cs.eventbench.data.GlobalDataContainer;
 import de.ugoe.cs.eventbench.web.WeblogParser;
+import de.ugoe.cs.eventbench.web.data.WebEvent;
 import de.ugoe.cs.util.console.Command;
 import de.ugoe.cs.util.console.Console;
@@ -38 +40 @@
                 int minLength = -1;
                 int maxLength = -1;
-                if( parameters.size()>=3 ) {
+                boolean generateFrequentUsers = false;
+                int frequentUserThreshold = 20;
+                if (parameters.size() >= 3) {
                         serverUrl = (String) parameters.get(2);
                 }
-                if (parameters.size() >= 5) {
+                if (parameters.size() >= 6) {
                         timeout = Integer.parseInt((String) parameters.get(3));
                         minLength = Integer.parseInt((String) parameters.get(4));
                         maxLength = Integer.parseInt((String) parameters.get(5));
                 }
+                if (parameters.size() >= 8) {
+                        generateFrequentUsers = Boolean.parseBoolean((String) parameters
+                                        .get(6));
+                        frequentUserThreshold = Integer
+                                        .parseInt((String) parameters.get(7));
+                }
 
                 WeblogParser parser = new WeblogParser();
-                if( serverUrl!=null ) {
+                if (serverUrl != null) {
                         parser.setUrl(serverUrl);
                 }
@@ -55 +65 @@
                         parser.setMinLength(minLength);
                         parser.setMaxLength(maxLength);
+                }
+                if (generateFrequentUsers) {
+                        parser.setFrequentUserThreshold(frequentUserThreshold);
                 }
                 try {
@@ -71 +84 @@
                         Console.traceln("Old data \"" + sequencesName + "\" overwritten");
                 }
+                if (generateFrequentUsers) {
+                        List<String> frequentUserIDs = parser.getFrequentUsers();
+                        List<Collection<List<WebEvent>>> frequentUserSessions = parser
+                                        .getFrequentUserSequences();
+                        for (int i = 0; i < frequentUserIDs.size(); i++) {
+                                GlobalDataContainer.getInstance().addData(
+                                                sequencesName + "_" + frequentUserIDs.get(i),
+                                                frequentUserSessions.get(i));
+                        }
+                }
         }
 
@@ -80 +103 @@
         @Override
         public void help() {
-                Console.println("Usage: loadSessionsFromClickstream <filename> <sequencesName> {<serverUrl>} {<timeout> <minSessionLength> <maxSessionLength>}");
+                Console.println("Usage: loadSessionsFromClickstream <filename> <sequencesName> {<serverUrl>} {<timeout> <minSessionLength> <maxSessionLength>} {<generateFrequentUsers> <frequentUserThreshold>}");
         }