diff --git a/core/pom.xml b/core/pom.xml index 7b87e77..4059f32 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -56,7 +56,7 @@ org.apache.solr - solr-solrj + solr-solrj 5.5.3 @@ -142,22 +142,10 @@ com.amazonaws - aws-java-sdk - 1.11.114 + aws-java-sdk-s3 + 1.11.166 - - - - com.paypal.sdk - paypal-core - 1.7.2 - - - com.paypal.sdk - rest-api-sdk - 1.13.0 - - + diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java b/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java new file mode 100644 index 0000000..a9f4fc8 --- /dev/null +++ b/core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java @@ -0,0 +1,27 @@ +package com.usvisatrack.core.dao.model; + +import com.usvisatrack.core.common.DataObject; + +/** + * USEmbassy ORM + * + * @author YuCheng Hu + * + */ +public class CheckeeVisa extends DataObject { + + private Long visaID; + + + public CheckeeVisa() { + } + + public Long getVisaID() { + return visaID; + } + + public void setVisaID(Long visaID) { + this.visaID = visaID; + } + +} diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/User.java b/core/src/main/java/com/usvisatrack/core/dao/model/User.java index b576540..44223ea 100644 --- a/core/src/main/java/com/usvisatrack/core/dao/model/User.java +++ b/core/src/main/java/com/usvisatrack/core/dao/model/User.java @@ -1,42 +1,25 @@ package com.usvisatrack.core.dao.model; -import java.util.ArrayList; import java.util.Date; -import java.util.List; import com.usvisatrack.core.common.DataObject; /** - * Merchant generated by hbm2java + * User ORM + * + * @author YuCheng + * */ -public class User extends DataObject implements java.io.Serializable { +public class User extends DataObject { - /** - * - */ - private static final long serialVersionUID = -8094138967079219902L; - - private String email; private String userName; private String password; - private String apiKey; - private String apiToken; - private String firstName; - private String lirstName; - private Date dateReg; - - + private String email; + private Date dateRegistered; public User() { - } - - public String getEmail() { - return email; - } - - public void setEmail(String email) { - this.email = email; + this.dateRegistered = new Date(); } public String getUserName() { @@ -55,46 +38,20 @@ public class User extends DataObject implements java.io.Serializable { this.password = password; } - public String getApiKey() { - return apiKey; + public String getEmail() { + return email; } - public void setApiKey(String apiKey) { - this.apiKey = apiKey; + public void setEmail(String email) { + this.email = email; } - public String getApiToken() { - return apiToken; + public Date getDateRegistered() { + return dateRegistered; } - public void setApiToken(String apiToken) { - this.apiToken = apiToken; + public void setDateRegistered(Date dateRegistered) { + this.dateRegistered = dateRegistered; } - public String getFirstName() { - return firstName; - } - - public void setFirstName(String firstName) { - this.firstName = firstName; - } - - public String getLirstName() { - return lirstName; - } - - public void setLirstName(String lirstName) { - this.lirstName = lirstName; - } - - public Date getDateReg() { - return dateReg; - } - - public void setDateReg(Date dateReg) { - this.dateReg = dateReg; - } - - - } diff --git a/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java b/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java index 5dfc6e6..b9ce202 100644 --- a/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java +++ b/core/src/main/java/com/usvisatrack/core/dao/model/Visa.java @@ -14,9 +14,10 @@ import com.usvisatrack.core.dao.model.data.VisaStatus; */ public class Visa extends DataObject { - private String ds160Code; + private User user; private USEmbassy usEmbassy; private VisaClass visaClass; + private String ds160Code; private VisaEntry visaEntry; private VisaStatus visaStatus; private String major; @@ -31,6 +32,14 @@ public class Visa extends DataObject { super.setModifyDate(date); } + public User getUser() { + return user; + } + + public void setUser(User user) { + this.user = user; + } + public String getDs160Code() { return ds160Code; } diff --git a/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java b/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java index 9cad908..1881c59 100644 --- a/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java +++ b/core/src/main/java/com/usvisatrack/core/factories/UserFactory.java @@ -1,16 +1,14 @@ package com.usvisatrack.core.factories; -import java.util.List; - import org.hibernate.Criteria; -import org.hibernate.Transaction; -import org.hibernate.criterion.Order; import org.hibernate.criterion.Restrictions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.usvisatrack.core.common.Factory; - +import com.usvisatrack.core.dao.model.USEmbassy; +import com.usvisatrack.core.dao.model.User; +import com.usvisatrack.core.dao.model.Visa; public class UserFactory { private static final int LISTING_PAGE_SIZE = 20; @@ -18,6 +16,58 @@ public class UserFactory { private static Logger logger = LoggerFactory.getLogger(UserFactory.class); + public static User get(Long id) { + logger.debug("Get User OBJ by ID - [{}]", id); + Factory.beginTransaction(); + User user = null; + try { + user = Factory.get(User.class, id); + Factory.commitTransaction(); + } catch (Exception ex) { + logger.error("Get User by user ID error", ex); + } finally { + Factory.rollbackTransaction(); + } + return user; + } + /** + * Get user by userName + * + * @param userName + * @return + */ + public static User get(String userName) { + logger.debug("Search USEmbassy - [{}]/[{}]", userName); + try { + Factory.beginTransaction(); + Criteria criteria = Factory.createCriteria(User.class); + criteria.add(Restrictions.eq("userName", userName)); + criteria.setMaxResults(1); + return (User) criteria.uniqueResult(); + } catch (Exception ex) { + logger.error("Search USEmbassy data error", ex); + } finally { + Factory.commitTransaction(); + } + + return null; + + } + + /** + * + * @param user + */ + public static void save(User user) { + try { + Factory.beginTransaction(); + Factory.saveOrUpdate(user); + Factory.commitTransaction(); + } catch (Exception ex) { + logger.error("Save USER OBJ ERROR", ex); + Factory.rollbackTransaction(); + } + } } \ No newline at end of file diff --git a/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java b/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java index f03dd62..9a9c5fa 100644 --- a/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java +++ b/core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.usvisatrack.core.common.Factory; +import com.usvisatrack.core.dao.model.CheckeeVisa; import com.usvisatrack.core.dao.model.Visa; import com.usvisatrack.core.dao.model.VisaClass; @@ -45,6 +46,34 @@ public class VisaFactory extends Factory { return item; } + /** + * Get Visa OBJ + * + * @param checkkeID + * @return + */ + public static Visa getVisaFromCheckee(Long checkkeID) { + logger.debug("Get Visa OBJ by checkkeID - [{}]", checkkeID); + Visa visa = null; + CheckeeVisa checkeeVisa = null; + + Factory.beginTransaction(); + + try { + checkeeVisa = Factory.get(CheckeeVisa.class, checkkeID); + if (checkeeVisa != null) { + visa = get(checkeeVisa.getVisaID()); + } + Factory.commitTransaction(); + } catch (Exception ex) { + logger.error("Get Visa by item ID error", ex); + } finally { + Factory.rollbackTransaction(); + } + + return visa; + } + /** * * @return @@ -80,4 +109,15 @@ public class VisaFactory extends Factory { } } + public static void save(CheckeeVisa checkeeVisa) { + try { + Factory.beginTransaction(); + Factory.saveOrUpdate(checkeeVisa); + Factory.commitTransaction(); + } catch (Exception ex) { + logger.error("Save CheckeeVisa OBJ ERROR", ex); + Factory.rollbackTransaction(); + } + } + } \ No newline at end of file diff --git a/core/src/main/resources/hbm/CheckeeVisa.hbm.xml b/core/src/main/resources/hbm/CheckeeVisa.hbm.xml new file mode 100644 index 0000000..9118eed --- /dev/null +++ b/core/src/main/resources/hbm/CheckeeVisa.hbm.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + diff --git a/core/src/main/resources/hbm/User.hbm.xml b/core/src/main/resources/hbm/User.hbm.xml index f8084a1..2b3a6db 100644 --- a/core/src/main/resources/hbm/User.hbm.xml +++ b/core/src/main/resources/hbm/User.hbm.xml @@ -3,34 +3,26 @@ "http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd"> - - - + + + - - - - - + - + - - + + - - - - - - + + diff --git a/core/src/main/resources/hbm/Visa.hbm.xml b/core/src/main/resources/hbm/Visa.hbm.xml index 4340a85..c37b10e 100644 --- a/core/src/main/resources/hbm/Visa.hbm.xml +++ b/core/src/main/resources/hbm/Visa.hbm.xml @@ -9,13 +9,18 @@ - - - + + + + + + + + com.usvisatrack.core.common.data.VisaStatus diff --git a/core/src/main/resources/hibernate.cfg.xml b/core/src/main/resources/hibernate.cfg.xml index 4c619d9..cbd3b1a 100644 --- a/core/src/main/resources/hibernate.cfg.xml +++ b/core/src/main/resources/hibernate.cfg.xml @@ -28,6 +28,7 @@ true + diff --git a/services/pom.xml b/services/pom.xml index 036b8ad..5240e8d 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -23,11 +23,6 @@ UTF-8 UTF-8 - - 4.0.3.RELEASE - 1.5.1.RELEASE - - 2.2 1.2 @@ -90,13 +85,6 @@ 1.4 - - - org.springframework - spring-test - ${spring-framework.version} - test - org.hamcrest @@ -141,12 +129,6 @@ geronimo-activation_1.1_spec 1.1 - - commons-logging - commons-logging - 1.1.3 - - org.apache.ws.commons.schema @@ -167,56 +149,10 @@ 1.2.3 - - commons-httpclient - commons-httpclient - 3.1 - - - - - org.apache.jclouds.labs - rackspace-cloudfiles-us - 1.9.0 - - - - - org.springframework - spring-core - ${spring-framework.version} - - - org.springframework - spring-beans - ${spring-framework.version} - - - org.springframework - spring-context - ${spring-framework.version} - - - org.springframework - spring-orm - ${spring-framework.version} - - - org.springframework - spring-aop - ${spring-framework.version} - - - org.springframework - spring-tx - ${spring-framework.version} - jaxen jaxen diff --git a/services/src/main/java/com/usvisatrack/services/VisaImporter.java b/services/src/main/java/com/usvisatrack/services/VisaImporter.java index c2cc2e0..aaf8bda 100644 --- a/services/src/main/java/com/usvisatrack/services/VisaImporter.java +++ b/services/src/main/java/com/usvisatrack/services/VisaImporter.java @@ -1,14 +1,15 @@ package com.usvisatrack.services; import java.io.IOException; -import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.jsoup.Jsoup; @@ -20,10 +21,12 @@ import org.slf4j.LoggerFactory; import com.usvisatrack.core.common.data.VisaEntry; import com.usvisatrack.core.common.data.VisaStatus; -import com.usvisatrack.core.dao.model.USEmbassy; +import com.usvisatrack.core.dao.model.CheckeeVisa; +import com.usvisatrack.core.dao.model.User; import com.usvisatrack.core.dao.model.Visa; import com.usvisatrack.core.dao.model.VisaClass; import com.usvisatrack.core.factories.USFactory; +import com.usvisatrack.core.factories.UserFactory; import com.usvisatrack.core.factories.VisaFactory; import com.usvisatrack.services.common.DataCrawl; @@ -36,9 +39,7 @@ import com.usvisatrack.services.common.DataCrawl; public class VisaImporter extends DataCrawl { private static final Logger logger = LoggerFactory.getLogger(VisaImporter.class); - public final static String ITEM_FTP_FOLDER = "/home/ftp/com-bcodepot/item"; - public final static String ITEM_PROCESSED_FOLDER = "/home/data/origin/com-bcodepot-www/processed/item"; - public final static String APIDATA_CDN_FOLDER = "/home/cdn/com-bcodepot-www/data/apidata"; + public final static String URL_CHECKEE = "https://www.checkee.info/main.php?dispdate="; public static HashMap visaClassMap = new HashMap(); @@ -50,7 +51,7 @@ public class VisaImporter extends DataCrawl { options.addOption("l", true, "Clean all agents input information"); options.addOption("h", true, "Clean one agent by input email address"); - CommandLineParser parser = new GnuParser(); + CommandLineParser parser = new DefaultParser(); // parse command line try { @@ -72,43 +73,70 @@ public class VisaImporter extends DataCrawl { @Override public void run() { initData(); // Init data from database - crawlWebVisa(); + crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime())); + crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime().minusMonths(1))); } /** * crawlWebItem by parse web page */ - private void crawlWebVisa() { + private void crawlWebVisa(String cURL) { logger.error("Crawl Web Data to load item info."); - List visaList = new ArrayList(); + + boolean isNewVisa = false; Document doc = null; try { - doc = Jsoup.connect("https://www.checkee.info/main.php?dispdate=2017-07").get(); + doc = Jsoup.connect(cURL).get(); Elements newsHeadlines = doc.select("table"); Element table = newsHeadlines.get(6); - int i = 0; + // int i = 0; + for (Element row : table.select("tr")) { - i++; + // i++; Elements tds = row.select("td"); if (!StringUtils.equalsIgnoreCase("ID", tds.get(1).text())) { - Visa visa = new Visa(); String checkeeCaseNumber = getCheckeeCaseNumber(tds.get(0)); + Visa visa = VisaFactory.getVisaFromCheckee(NumberUtils.toLong(checkeeCaseNumber)); + + if (visa == null) { + visa = new Visa(); + isNewVisa = true; + } + // SET VISA CLASS String visaClassName = StringUtils.trimToEmpty(tds.get(2).text()); if (StringUtils.isNotBlank(visaClassName)) { switch (visaClassName) { + case "B1": + visaClassName = "B-1"; + break; + case "B2": + visaClassName = "B-2"; + break; case "H1": visaClassName = "H1-B"; break; case "H4": visaClassName = "H-4"; break; + case "F1": + visaClassName = "F-1"; + break; + case "F2": + visaClassName = "F-2"; + break; + case "J1": + visaClassName = "J-1"; + break; + case "L1": + visaClassName = "L-1"; + break; } visa.setVisaClass(visaClassMap.get(visaClassName)); @@ -132,11 +160,26 @@ public class VisaImporter extends DataCrawl { String usEmbassyName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(4).text())); if (StringUtils.isNotBlank(usEmbassyName)) { switch (usEmbassyName) { + case "BEIJING": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "BeiJing")); + break; + case "CHENGDU": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Chengdu")); + break; case "GUANGZHOU": visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Guangzhou")); break; - case "RENEWAL": - visa.setVisaEntry(VisaEntry.RENEWAL); + case "SHANGHAI": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shanghai")); + break; + case "SHENYANG": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shenyang")); + break; + case "WUHAN": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Wuhan")); + break; + case "HONGKONG": + visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "HongKong")); break; } @@ -166,21 +209,25 @@ public class VisaImporter extends DataCrawl { if (StringUtils.isNotBlank(dateVisaInterview)) { visa.setDateVisaInterview(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaInterview).toDate()); } - if (StringUtils.isNotBlank(dateVisaIssued)) { + if (StringUtils.isNotBlank(dateVisaIssued) && !StringUtils.equals(dateVisaIssued, "0000-00-00")) { visa.setDateVisaIssued(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaIssued).toDate()); } - Element link = tds.get(10).select("a").first(); - - logger.debug(">>>>>>>>>>>>>>>[{}]", link.attr("href")); - updateVisaNote(visa, checkeeCaseNumber); - + visa.setModifyDate(new Date()); + visa.setUser(getUser(checkeeCaseNumber)); VisaFactory.save(visa); + + if (isNewVisa) { + CheckeeVisa checkeeVisa = new CheckeeVisa(); + checkeeVisa.setId(NumberUtils.toLong(checkeeCaseNumber)); + checkeeVisa.setVisaID(visa.getId()); + VisaFactory.save(checkeeVisa); + } } - if (i == 2) - break; + // if (i == 2) + // break; } @@ -213,13 +260,48 @@ public class VisaImporter extends DataCrawl { */ private String getCheckeeCaseNumber(Element element) { String checkeeCaseNumber = null; - Element link = element.select("a").first(); checkeeCaseNumber = StringUtils.substringAfterLast(link.attr("href"), "casenum="); return checkeeCaseNumber; } + /** + * + * @param userName + * @return + */ + private User getUser(String checkeeCaseNumber) { + String userName = null; + String userEmail = null; + + User user = null; + Document doc = null; + + try { + doc = Jsoup.connect("https://www.checkee.info/update.php?casenum=" + checkeeCaseNumber).get(); + Elements elements = doc.select("input[name=email_dis]"); + userEmail = elements.first().val(); + + elements = doc.select("b"); + userName = StringUtils.trim(StringUtils.substringAfter(elements.get(7).text(), "ID:")); + + user = UserFactory.get(userName); + if (user == null) { + user = new User(); + user.setUserName(userName); + user.setEmail(userEmail); + UserFactory.save(user); + } + + } catch (Exception e) { + // TODO: handle exception + e.printStackTrace(); + } + + return user; + } + private void updateVisaNote(Visa visa, String checkeeCaseNumber) { Document doc = null; diff --git a/services/src/main/resources/hibernate.properties b/services/src/main/resources/hibernate.properties index a3697d1..e84e56e 100644 --- a/services/src/main/resources/hibernate.properties +++ b/services/src/main/resources/hibernate.properties @@ -1,7 +1,7 @@ # JDBC Connection hibernate.connection.driver_class=org.mariadb.jdbc.Driver hibernate.dialect=org.hibernate.dialect.MySQL5Dialect -hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8&autoReconnect=true +hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8MB64&autoReconnect=true # Username and Password hibernate.connection.username=usvisatrack.web