diff --git a/core/pom.xml b/core/pom.xml
index 7b87e77..4059f32 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -56,7 +56,7 @@
org.apache.solr
- solr-solrj
+ solr-solrj
5.5.3
@@ -142,22 +142,10 @@
com.amazonaws
- aws-java-sdk
- 1.11.114
+ aws-java-sdk-s3
+ 1.11.166
-
-
-
- com.paypal.sdk
- paypal-core
- 1.7.2
-
-
- com.paypal.sdk
- rest-api-sdk
- 1.13.0
-
-
+
diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java b/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java
new file mode 100644
index 0000000..a9f4fc8
--- /dev/null
+++ b/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java
@@ -0,0 +1,27 @@
+package com.usvisatrack.core.dao.model;
+
+import com.usvisatrack.core.common.DataObject;
+
+/**
+ * USEmbassy ORM
+ *
+ * @author YuCheng Hu
+ *
+ */
+public class CheckeeVisa extends DataObject {
+
+ private Long visaID;
+
+
+ public CheckeeVisa() {
+ }
+
+ public Long getVisaID() {
+ return visaID;
+ }
+
+ public void setVisaID(Long visaID) {
+ this.visaID = visaID;
+ }
+
+}
diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/User.java b/core/src/main/java/com/usvisatrack/core/dao/model/User.java
index b576540..44223ea 100644
--- a/core/src/main/java/com/usvisatrack/core/dao/model/User.java
+++ b/core/src/main/java/com/usvisatrack/core/dao/model/User.java
@@ -1,42 +1,25 @@
package com.usvisatrack.core.dao.model;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.List;
import com.usvisatrack.core.common.DataObject;
/**
- * Merchant generated by hbm2java
+ * User ORM
+ *
+ * @author YuCheng
+ *
*/
-public class User extends DataObject implements java.io.Serializable {
+public class User extends DataObject {
- /**
- *
- */
- private static final long serialVersionUID = -8094138967079219902L;
-
- private String email;
private String userName;
private String password;
- private String apiKey;
- private String apiToken;
- private String firstName;
- private String lirstName;
- private Date dateReg;
-
-
+ private String email;
+ private Date dateRegistered;
public User() {
- }
-
- public String getEmail() {
- return email;
- }
-
- public void setEmail(String email) {
- this.email = email;
+ this.dateRegistered = new Date();
}
public String getUserName() {
@@ -55,46 +38,20 @@ public class User extends DataObject implements java.io.Serializable {
this.password = password;
}
- public String getApiKey() {
- return apiKey;
+ public String getEmail() {
+ return email;
}
- public void setApiKey(String apiKey) {
- this.apiKey = apiKey;
+ public void setEmail(String email) {
+ this.email = email;
}
- public String getApiToken() {
- return apiToken;
+ public Date getDateRegistered() {
+ return dateRegistered;
}
- public void setApiToken(String apiToken) {
- this.apiToken = apiToken;
+ public void setDateRegistered(Date dateRegistered) {
+ this.dateRegistered = dateRegistered;
}
- public String getFirstName() {
- return firstName;
- }
-
- public void setFirstName(String firstName) {
- this.firstName = firstName;
- }
-
- public String getLirstName() {
- return lirstName;
- }
-
- public void setLirstName(String lirstName) {
- this.lirstName = lirstName;
- }
-
- public Date getDateReg() {
- return dateReg;
- }
-
- public void setDateReg(Date dateReg) {
- this.dateReg = dateReg;
- }
-
-
-
}
diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java b/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java
index 5dfc6e6..b9ce202 100644
--- a/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java
+++ b/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java
@@ -14,9 +14,10 @@ import com.usvisatrack.core.dao.model.data.VisaStatus;
*/
public class Visa extends DataObject {
- private String ds160Code;
+ private User user;
private USEmbassy usEmbassy;
private VisaClass visaClass;
+ private String ds160Code;
private VisaEntry visaEntry;
private VisaStatus visaStatus;
private String major;
@@ -31,6 +32,14 @@ public class Visa extends DataObject {
super.setModifyDate(date);
}
+ public User getUser() {
+ return user;
+ }
+
+ public void setUser(User user) {
+ this.user = user;
+ }
+
public String getDs160Code() {
return ds160Code;
}
diff --git a/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java b/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java
index 9cad908..1881c59 100644
--- a/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java
+++ b/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java
@@ -1,16 +1,14 @@
package com.usvisatrack.core.factories;
-import java.util.List;
-
import org.hibernate.Criteria;
-import org.hibernate.Transaction;
-import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.Factory;
-
+import com.usvisatrack.core.dao.model.USEmbassy;
+import com.usvisatrack.core.dao.model.User;
+import com.usvisatrack.core.dao.model.Visa;
public class UserFactory {
private static final int LISTING_PAGE_SIZE = 20;
@@ -18,6 +16,58 @@ public class UserFactory {
private static Logger logger = LoggerFactory.getLogger(UserFactory.class);
+ public static User get(Long id) {
+ logger.debug("Get User OBJ by ID - [{}]", id);
+ Factory.beginTransaction();
+ User user = null;
+ try {
+ user = Factory.get(User.class, id);
+ Factory.commitTransaction();
+ } catch (Exception ex) {
+ logger.error("Get User by user ID error", ex);
+ } finally {
+ Factory.rollbackTransaction();
+ }
+ return user;
+ }
+ /**
+ * Get user by userName
+ *
+ * @param userName
+ * @return
+ */
+ public static User get(String userName) {
+ logger.debug("Search USEmbassy - [{}]/[{}]", userName);
+ try {
+ Factory.beginTransaction();
+ Criteria criteria = Factory.createCriteria(User.class);
+ criteria.add(Restrictions.eq("userName", userName));
+ criteria.setMaxResults(1);
+ return (User) criteria.uniqueResult();
+ } catch (Exception ex) {
+ logger.error("Search USEmbassy data error", ex);
+ } finally {
+ Factory.commitTransaction();
+ }
+
+ return null;
+
+ }
+
+ /**
+ *
+ * @param user
+ */
+ public static void save(User user) {
+ try {
+ Factory.beginTransaction();
+ Factory.saveOrUpdate(user);
+ Factory.commitTransaction();
+ } catch (Exception ex) {
+ logger.error("Save USER OBJ ERROR", ex);
+ Factory.rollbackTransaction();
+ }
+ }
}
\ No newline at end of file
diff --git a/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java b/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java
index f03dd62..9a9c5fa 100644
--- a/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java
+++ b/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java
@@ -7,6 +7,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.Factory;
+import com.usvisatrack.core.dao.model.CheckeeVisa;
import com.usvisatrack.core.dao.model.Visa;
import com.usvisatrack.core.dao.model.VisaClass;
@@ -45,6 +46,34 @@ public class VisaFactory extends Factory {
return item;
}
+ /**
+ * Get Visa OBJ
+ *
+ * @param checkkeID
+ * @return
+ */
+ public static Visa getVisaFromCheckee(Long checkkeID) {
+ logger.debug("Get Visa OBJ by checkkeID - [{}]", checkkeID);
+ Visa visa = null;
+ CheckeeVisa checkeeVisa = null;
+
+ Factory.beginTransaction();
+
+ try {
+ checkeeVisa = Factory.get(CheckeeVisa.class, checkkeID);
+ if (checkeeVisa != null) {
+ visa = get(checkeeVisa.getVisaID());
+ }
+ Factory.commitTransaction();
+ } catch (Exception ex) {
+ logger.error("Get Visa by item ID error", ex);
+ } finally {
+ Factory.rollbackTransaction();
+ }
+
+ return visa;
+ }
+
/**
*
* @return
@@ -80,4 +109,15 @@ public class VisaFactory extends Factory {
}
}
+ public static void save(CheckeeVisa checkeeVisa) {
+ try {
+ Factory.beginTransaction();
+ Factory.saveOrUpdate(checkeeVisa);
+ Factory.commitTransaction();
+ } catch (Exception ex) {
+ logger.error("Save CheckeeVisa OBJ ERROR", ex);
+ Factory.rollbackTransaction();
+ }
+ }
+
}
\ No newline at end of file
diff --git a/core/src/main/resources/hbm/CheckeeVisa.hbm.xml b/core/src/main/resources/hbm/CheckeeVisa.hbm.xml
new file mode 100644
index 0000000..9118eed
--- /dev/null
+++ b/core/src/main/resources/hbm/CheckeeVisa.hbm.xml
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/core/src/main/resources/hbm/User.hbm.xml b/core/src/main/resources/hbm/User.hbm.xml
index f8084a1..2b3a6db 100644
--- a/core/src/main/resources/hbm/User.hbm.xml
+++ b/core/src/main/resources/hbm/User.hbm.xml
@@ -3,34 +3,26 @@
"http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd">
-
-
-
+
+
+
-
-
-
-
-
+
-
+
-
-
+
+
-
-
-
-
-
-
+
+
diff --git a/core/src/main/resources/hbm/Visa.hbm.xml b/core/src/main/resources/hbm/Visa.hbm.xml
index 4340a85..c37b10e 100644
--- a/core/src/main/resources/hbm/Visa.hbm.xml
+++ b/core/src/main/resources/hbm/Visa.hbm.xml
@@ -9,13 +9,18 @@
-
-
-
+
+
+
+
+
+
+
+
com.usvisatrack.core.common.data.VisaStatus
diff --git a/core/src/main/resources/hibernate.cfg.xml b/core/src/main/resources/hibernate.cfg.xml
index 4c619d9..cbd3b1a 100644
--- a/core/src/main/resources/hibernate.cfg.xml
+++ b/core/src/main/resources/hibernate.cfg.xml
@@ -28,6 +28,7 @@
true
+
diff --git a/services/pom.xml b/services/pom.xml
index 036b8ad..5240e8d 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -23,11 +23,6 @@
UTF-8
UTF-8
-
- 4.0.3.RELEASE
- 1.5.1.RELEASE
-
-
2.2
1.2
@@ -90,13 +85,6 @@
1.4
-
-
- org.springframework
- spring-test
- ${spring-framework.version}
- test
-
org.hamcrest
@@ -141,12 +129,6 @@
geronimo-activation_1.1_spec
1.1
-
- commons-logging
- commons-logging
- 1.1.3
-
-
org.apache.ws.commons.schema
@@ -167,56 +149,10 @@
1.2.3
-
- commons-httpclient
- commons-httpclient
- 3.1
-
-
-
-
- org.apache.jclouds.labs
- rackspace-cloudfiles-us
- 1.9.0
-
-
-
-
- org.springframework
- spring-core
- ${spring-framework.version}
-
-
- org.springframework
- spring-beans
- ${spring-framework.version}
-
-
- org.springframework
- spring-context
- ${spring-framework.version}
-
-
- org.springframework
- spring-orm
- ${spring-framework.version}
-
-
- org.springframework
- spring-aop
- ${spring-framework.version}
-
-
- org.springframework
- spring-tx
- ${spring-framework.version}
-
jaxen
jaxen
diff --git a/services/src/main/java/com/usvisatrack/services/VisaImporter.java b/services/src/main/java/com/usvisatrack/services/VisaImporter.java
index c2cc2e0..aaf8bda 100644
--- a/services/src/main/java/com/usvisatrack/services/VisaImporter.java
+++ b/services/src/main/java/com/usvisatrack/services/VisaImporter.java
@@ -1,14 +1,15 @@
package com.usvisatrack.services;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.jsoup.Jsoup;
@@ -20,10 +21,12 @@ import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.data.VisaEntry;
import com.usvisatrack.core.common.data.VisaStatus;
-import com.usvisatrack.core.dao.model.USEmbassy;
+import com.usvisatrack.core.dao.model.CheckeeVisa;
+import com.usvisatrack.core.dao.model.User;
import com.usvisatrack.core.dao.model.Visa;
import com.usvisatrack.core.dao.model.VisaClass;
import com.usvisatrack.core.factories.USFactory;
+import com.usvisatrack.core.factories.UserFactory;
import com.usvisatrack.core.factories.VisaFactory;
import com.usvisatrack.services.common.DataCrawl;
@@ -36,9 +39,7 @@ import com.usvisatrack.services.common.DataCrawl;
public class VisaImporter extends DataCrawl {
private static final Logger logger = LoggerFactory.getLogger(VisaImporter.class);
- public final static String ITEM_FTP_FOLDER = "/home/ftp/com-bcodepot/item";
- public final static String ITEM_PROCESSED_FOLDER = "/home/data/origin/com-bcodepot-www/processed/item";
- public final static String APIDATA_CDN_FOLDER = "/home/cdn/com-bcodepot-www/data/apidata";
+ public final static String URL_CHECKEE = "https://www.checkee.info/main.php?dispdate=";
public static HashMap visaClassMap = new HashMap();
@@ -50,7 +51,7 @@ public class VisaImporter extends DataCrawl {
options.addOption("l", true, "Clean all agents input information");
options.addOption("h", true, "Clean one agent by input email address");
- CommandLineParser parser = new GnuParser();
+ CommandLineParser parser = new DefaultParser();
// parse command line
try {
@@ -72,43 +73,70 @@ public class VisaImporter extends DataCrawl {
@Override
public void run() {
initData(); // Init data from database
- crawlWebVisa();
+ crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime()));
+ crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime().minusMonths(1)));
}
/**
* crawlWebItem by parse web page
*/
- private void crawlWebVisa() {
+ private void crawlWebVisa(String cURL) {
logger.error("Crawl Web Data to load item info.");
- List visaList = new ArrayList();
+
+ boolean isNewVisa = false;
Document doc = null;
try {
- doc = Jsoup.connect("https://www.checkee.info/main.php?dispdate=2017-07").get();
+ doc = Jsoup.connect(cURL).get();
Elements newsHeadlines = doc.select("table");
Element table = newsHeadlines.get(6);
- int i = 0;
+ // int i = 0;
+
for (Element row : table.select("tr")) {
- i++;
+ // i++;
Elements tds = row.select("td");
if (!StringUtils.equalsIgnoreCase("ID", tds.get(1).text())) {
- Visa visa = new Visa();
String checkeeCaseNumber = getCheckeeCaseNumber(tds.get(0));
+ Visa visa = VisaFactory.getVisaFromCheckee(NumberUtils.toLong(checkeeCaseNumber));
+
+ if (visa == null) {
+ visa = new Visa();
+ isNewVisa = true;
+ }
+
// SET VISA CLASS
String visaClassName = StringUtils.trimToEmpty(tds.get(2).text());
if (StringUtils.isNotBlank(visaClassName)) {
switch (visaClassName) {
+ case "B1":
+ visaClassName = "B-1";
+ break;
+ case "B2":
+ visaClassName = "B-2";
+ break;
case "H1":
visaClassName = "H1-B";
break;
case "H4":
visaClassName = "H-4";
break;
+ case "F1":
+ visaClassName = "F-1";
+ break;
+ case "F2":
+ visaClassName = "F-2";
+ break;
+ case "J1":
+ visaClassName = "J-1";
+ break;
+ case "L1":
+ visaClassName = "L-1";
+ break;
}
visa.setVisaClass(visaClassMap.get(visaClassName));
@@ -132,11 +160,26 @@ public class VisaImporter extends DataCrawl {
String usEmbassyName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(4).text()));
if (StringUtils.isNotBlank(usEmbassyName)) {
switch (usEmbassyName) {
+ case "BEIJING":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "BeiJing"));
+ break;
+ case "CHENGDU":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Chengdu"));
+ break;
case "GUANGZHOU":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Guangzhou"));
break;
- case "RENEWAL":
- visa.setVisaEntry(VisaEntry.RENEWAL);
+ case "SHANGHAI":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shanghai"));
+ break;
+ case "SHENYANG":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shenyang"));
+ break;
+ case "WUHAN":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Wuhan"));
+ break;
+ case "HONGKONG":
+ visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "HongKong"));
break;
}
@@ -166,21 +209,25 @@ public class VisaImporter extends DataCrawl {
if (StringUtils.isNotBlank(dateVisaInterview)) {
visa.setDateVisaInterview(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaInterview).toDate());
}
- if (StringUtils.isNotBlank(dateVisaIssued)) {
+ if (StringUtils.isNotBlank(dateVisaIssued) && !StringUtils.equals(dateVisaIssued, "0000-00-00")) {
visa.setDateVisaIssued(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaIssued).toDate());
}
- Element link = tds.get(10).select("a").first();
-
- logger.debug(">>>>>>>>>>>>>>>[{}]", link.attr("href"));
-
updateVisaNote(visa, checkeeCaseNumber);
-
+ visa.setModifyDate(new Date());
+ visa.setUser(getUser(checkeeCaseNumber));
VisaFactory.save(visa);
+
+ if (isNewVisa) {
+ CheckeeVisa checkeeVisa = new CheckeeVisa();
+ checkeeVisa.setId(NumberUtils.toLong(checkeeCaseNumber));
+ checkeeVisa.setVisaID(visa.getId());
+ VisaFactory.save(checkeeVisa);
+ }
}
- if (i == 2)
- break;
+ // if (i == 2)
+ // break;
}
@@ -213,13 +260,48 @@ public class VisaImporter extends DataCrawl {
*/
private String getCheckeeCaseNumber(Element element) {
String checkeeCaseNumber = null;
-
Element link = element.select("a").first();
checkeeCaseNumber = StringUtils.substringAfterLast(link.attr("href"), "casenum=");
return checkeeCaseNumber;
}
+ /**
+ *
+ * @param userName
+ * @return
+ */
+ private User getUser(String checkeeCaseNumber) {
+ String userName = null;
+ String userEmail = null;
+
+ User user = null;
+ Document doc = null;
+
+ try {
+ doc = Jsoup.connect("https://www.checkee.info/update.php?casenum=" + checkeeCaseNumber).get();
+ Elements elements = doc.select("input[name=email_dis]");
+ userEmail = elements.first().val();
+
+ elements = doc.select("b");
+ userName = StringUtils.trim(StringUtils.substringAfter(elements.get(7).text(), "ID:"));
+
+ user = UserFactory.get(userName);
+ if (user == null) {
+ user = new User();
+ user.setUserName(userName);
+ user.setEmail(userEmail);
+ UserFactory.save(user);
+ }
+
+ } catch (Exception e) {
+ // TODO: handle exception
+ e.printStackTrace();
+ }
+
+ return user;
+ }
+
private void updateVisaNote(Visa visa, String checkeeCaseNumber) {
Document doc = null;
diff --git a/services/src/main/resources/hibernate.properties b/services/src/main/resources/hibernate.properties
index a3697d1..e84e56e 100644
--- a/services/src/main/resources/hibernate.properties
+++ b/services/src/main/resources/hibernate.properties
@@ -1,7 +1,7 @@
# JDBC Connection
hibernate.connection.driver_class=org.mariadb.jdbc.Driver
hibernate.dialect=org.hibernate.dialect.MySQL5Dialect
-hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8&autoReconnect=true
+hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8MB64&autoReconnect=true
# Username and Password
hibernate.connection.username=usvisatrack.web