diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsRecordReportController.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsRecordReportController.java new file mode 100644 index 0000000..b6c7edc --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsRecordReportController.java @@ -0,0 +1,156 @@ +package net.hzvtc.es.controller; + +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.utils.file.FileUtils; +import net.hzvtc.service.EsRecordReportService; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Map; + +@RestController +@RequestMapping("/es/result/report") +@Slf4j +@RequiredArgsConstructor +public class EsRecordReportController { + private final EsRecordReportService esRecordReportService; + + @PostMapping("/getBirdStatData") + public AjaxResult getBirdStatData(@RequestBody JSONObject object) { + return AjaxResult.success(esRecordReportService.getBirdDailyReportStatByParam(object)); + } + + @PostMapping("/downloadDetailsReport") + public void downloadReport(@RequestParam Map param, HttpServletResponse response, HttpServletRequest request) { + +// obj.put("dateFrom", param.get("dateFrom")); +// obj.put("dateTo", param.get("dateTo")); +// obj.put("excludeIds", param.get("excludeIds")); +// log.info("=============={}", obj); + String filePath = esRecordReportService.exportResultRecordListFileByParam(param); +// log.info("===========fileName============{}", filePath); + if (filePath != null) { + try { + String realFileName = System.currentTimeMillis() + ".xlsx"; + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + FileUtils.setAttachmentResponseHeader(response, realFileName); + FileUtils.writeBytes(filePath, response.getOutputStream()); + } catch (Exception ex) { + log.error("error", ex); + } + } + } + + // +// @PostMapping("/sendEmail") +// public AjaxResult sendEmail(@RequestBody JSONObject object) { +// return AjaxResult.success(esRecordReportService.sendBirdDataStatEmail(object)); +// } + @PostMapping("/speciesMediaStatList") + public JSONObject getSpeciesMediaListByParam(@RequestBody JSONObject object) { + return esRecordReportService.getSpeciesMediaListByParam(object); + } + + @PostMapping("/recordStatList") + public JSONObject getRecordStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getResultRecordStatReportByParam(object); + } + + @PostMapping("/areaStatList") + public JSONObject getAreaStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getAreaStatReportByParam(object); + } + + @PostMapping("/dateStatList") + public JSONObject getRecordStatByDate(@RequestBody JSONObject object) { + return esRecordReportService.getRecordStatDateReportByParam(object); + } + + @PostMapping("/dateValidStatList") + public JSONObject getValidStatByDate(@RequestBody JSONObject object) { + return esRecordReportService.getValidStatDateReportByParam(object); + } + + @PostMapping("/recordValidStatList") + public JSONObject getRecordValidStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getResultRecordValidStatReportByParam(object); + } + + @PostMapping("/heatStatList") + public JSONObject getRecordValidHeataStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getHeatValidStatReportByParam(object); + } + + @PostMapping("/recordDailyStatList") + public JSONObject getRecordDailyStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getRecordDailyStatReportByParam(object); + } + + @PostMapping("/cameraStatList") + public JSONObject getCameraStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getCameraRecordStatReportByParam(object); + } + + @PostMapping("/habitatStatList") + public JSONObject getHabitatStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getSpeciesHabitatStatReportByParam(object); + } + + @PostMapping("/raiStatList") + public JSONObject getRAIStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getValidRAIReportByParam(object); + } + + @PostMapping("/rnStatList") + public JSONObject getRNStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getValidRNReportByParam(object); + } + + @PostMapping("/percentStatList") + public JSONObject getPercentStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getValidPercentReportByParam(object); + } + + @PostMapping("/rrpStatList") + public JSONObject getSpeciesRRPStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getValidRAIRNPtReportByParam(object); + } + + @PostMapping("/rhythmStatList") + public JSONObject getRhythmStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getRhythmReportByParam(object); + } + + @PostMapping("/multiStatList") + public JSONObject getMultiStatByParam(@RequestBody JSONObject object) { + return esRecordReportService.getSpeciesMultiStatReportByParam(object); + } + + @GetMapping("/speciesList") + public JSONObject getSpeciesList() { + return esRecordReportService.getSpeciesTreeList(); + } + + @PostMapping("/periodStat") + public AjaxResult getBirdPeriodStat(@RequestBody JSONObject obj) { + obj.put("type", 1); + return AjaxResult.success(esRecordReportService.getBirdPeriodStatByParam(obj)); + } + + @PostMapping("/monthStat") + public AjaxResult getMonthStat(@RequestBody JSONObject obj) { + obj.put("type", 2); + return AjaxResult.success(esRecordReportService.getBirdPeriodStatByParam(obj)); + } + + @PostMapping("/yearStat") + public AjaxResult getYearStat(@RequestBody JSONObject obj) { + obj.put("type", 3); + return AjaxResult.success(esRecordReportService.getBirdPeriodStatByParam(obj)); + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsResultRecordDetailsController.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsResultRecordDetailsController.java new file mode 100644 index 0000000..836f57e --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/controller/EsResultRecordDetailsController.java @@ -0,0 +1,148 @@ +package net.hzvtc.es.controller; + +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.core.page.TableDataInfo; +import net.hzvtc.device.domain.DeviceInfo; +import net.hzvtc.service.EsResultRecordDetailsService; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RestController +@RequestMapping("/es/result/details") +@RequiredArgsConstructor +public class EsResultRecordDetailsController { + private final EsResultRecordDetailsService esResultRecordDetailsService; + + + @PostMapping("/querySpeciesByParam") + public TableDataInfo querySpeciesListByParam(@RequestBody JSONObject params) { + params.put("isDeleted", 0); + return esResultRecordDetailsService.querySpeciesRecordListByParam(params, 0); + } + + @PostMapping("/getSpeciesStatInfoByParam") + public JSONObject getSpeciesStatInfoByParam(@RequestBody JSONObject params) { + return esResultRecordDetailsService.getSpeciesStatInfo(params); + } + + @PostMapping("/querySpeciesByParamForApp") + public TableDataInfo querySpeciesListByParamForApp(@RequestBody JSONObject params) { + params.put("isDeleted", 0); + return esResultRecordDetailsService.querySpeciesRecordListByParam(params, 1); + } + + @PostMapping("/querySpeciesByQuery") + public TableDataInfo querySpeciesListByQuery(@RequestBody JSONObject obj) { + obj.put("isDeleted", 0); + return esResultRecordDetailsService.getProtectedAreaSpeciesByQuery(obj); + } + + @PostMapping("/birdResultMedias") + public TableDataInfo getBirdResultMediasByParam(@RequestBody JSONObject params) { + return esResultRecordDetailsService.getResultMediaListByParam(params); + } + + @DeleteMapping("/delDocument/{ids}") + public AjaxResult deleteDocumentById(@PathVariable String ids) { + return AjaxResult.success(esResultRecordDetailsService.deleteDocumentById(ids)); + } + + + @PostMapping("/updateResultRecord") + public AjaxResult updateResultRecord(@RequestBody JSONObject data) { + return AjaxResult.success(esResultRecordDetailsService.updateResultRecordInfoById(data)); + } + + @PostMapping("/updateResultRecords") + public AjaxResult updateResultRecords(@RequestBody JSONObject data) { + return AjaxResult.success(esResultRecordDetailsService.updateResultRecordInfoByIds(data)); + } + + @PostMapping("/birdAudioDailyStat") + public AjaxResult getBirdAudioDailyStatList(@RequestBody JSONObject obj) { + obj.put("isDeleted", 0); + return AjaxResult.success(esResultRecordDetailsService.getBirdAudioDailyStatListByParam(obj)); + } + + @PostMapping("/birdDailyStat") + public AjaxResult getBirdDailyStatList(@RequestBody JSONObject obj) { + obj.put("isDeleted", 0); + return AjaxResult.success(esResultRecordDetailsService.getBirdDailyStatListByParam(obj)); + } + + @PostMapping("/birdAudioMonthStat") + public AjaxResult getBirdAudioMonthStatList(@RequestBody JSONObject obj) { + obj.put("isDeleted", 0); + return AjaxResult.success(esResultRecordDetailsService.getBirdAudioMonthStatListByParam(obj)); + } + + @PostMapping("/birdAudioYearStat") + public AjaxResult getBirdAudioYearStatList(@RequestBody JSONObject obj) { + obj.put("isDeleted", 0); + return AjaxResult.success(esResultRecordDetailsService.getBirdAudioYearStatListByParam(obj)); + } + + @GetMapping("/listByParam") + public AjaxResult getListByParam(DeviceInfo deviceInfo) { + return AjaxResult.success(esResultRecordDetailsService.getDeviceInfoListForPage(deviceInfo)); + } + + @GetMapping("/speciesListByParam") + public AjaxResult getSpeciesListByParam() { + return AjaxResult.success(esResultRecordDetailsService.getSpeciesListForSelect()); + } + + @GetMapping("/mediaStatInfo") + public JSONObject getStatAmount() { + return esResultRecordDetailsService.getResultMediaStatByParam(); + } + + @PostMapping("/speciesStatList") + public JSONObject getSpeciesMediaStatInfoByParam(@RequestBody JSONObject object) { + return esResultRecordDetailsService.getSpeciesStatListByParam(object); + } + + @GetMapping("/listForMap") + public List getDeviceListForGisMap() { + return esResultRecordDetailsService.getDeviceInfoListForGisMap(); + } + + @PostMapping("/updateDocumentInfoByParam") + public AjaxResult updateStatus(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.updateResultRecordInfoById(param)); + } + + @PostMapping("/updateDocumentFavorite") + public AjaxResult updateFavorite(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.updateResultRecordInfoById(param)); + } + + @PostMapping("/updateSpeciesInfo") + public AjaxResult updateSpeciesInfo(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.updateSpeciesInfo(param)); + } + + @PostMapping("/deleteSpeciesInfo") + public AjaxResult deleteSpeciesInfo(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.deleteSpeciesInfo(param)); + } + + @PostMapping("/addSpeciesInfo") + public AjaxResult addSpeciesInfo(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.addSpeciesInfo(param)); + } + + @PostMapping("/favoriteList") + public AjaxResult getFavoriteMediaList(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.getFavoriteMediaList(param)); + } + + @PostMapping("/listByItemId") + public AjaxResult getMediaListByItemId(@RequestBody JSONObject param) { + return AjaxResult.success(esResultRecordDetailsService.getMediaListByItemId(param)); + } +} + diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetails.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetails.java new file mode 100644 index 0000000..a9bc854 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetails.java @@ -0,0 +1,64 @@ +package net.hzvtc.es.domain; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.ibatis.type.Alias; +import org.springframework.data.annotation.Id; + +import java.time.ZonedDateTime; +import java.util.List; + +@Data +@Alias("EsResultRecordDetails") +@AllArgsConstructor +@NoArgsConstructor +//@JsonIgnoreProperties(ignoreUnknown = true) +public class EsResultRecordDetails { + @Id + private Long id; + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "Asia/Shanghai") + private ZonedDateTime recordTime; + + private Long deviceId; + + private String deviceName; + + private Integer deviceType; + + private String duid; + + private Long deptId; + + private String deptName; + + private String deptLetter; + + private int checked; + + private Long mediaId; + + private String mediaUrl; + + private String metaUrl; + + private int mediaType; + + private Integer favorite; + + private String memo; + + private String tag; + + private String version; + + private Long itemId; + + private List items; + + private int isDeleted; + + private String aiName; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsBak.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsBak.java new file mode 100644 index 0000000..925881f --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsBak.java @@ -0,0 +1,59 @@ +package net.hzvtc.es.domain; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.springframework.data.annotation.Id; + +import java.time.ZonedDateTime; + + +public class EsResultRecordDetailsBak { + @Id + private Long id; + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "Asia/Shanghai") + private ZonedDateTime recordTime; + + private Long recordId; + + private Long deviceId; + + private String deviceName; + + private Integer deviceType; + + private String duid; + + private Long speciesId; + + private String speciesName; + + private String scientificName; + + private Long deptId; + + private int checked; + + private int count; + + private String habitat; + + private double confidence; + + private Long mediaId; + + private String mediaUrl; + + private String metaUrl; + + private int mediaType; + + private int recordType; + + private Integer favorite; + + private String birdIcon; + + private String version; + + private int isDeleted; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsVo.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsVo.java new file mode 100644 index 0000000..7159491 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordDetailsVo.java @@ -0,0 +1,86 @@ +package net.hzvtc.es.domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import org.apache.ibatis.type.Alias; +import org.springframework.data.annotation.Id; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +@Data +@Alias("EsResultRecordDetailsVo") +@JsonIgnoreProperties(ignoreUnknown = true) +public class EsResultRecordDetailsVo { + @Id + private Long id; + + @JsonIgnore + private LocalDateTime recordTime1; + + private Long recordId; + + private Long deviceId; + + private String deviceName; + + private String duid; + + private Long speciesId; + + private String speciesName; + + private String scientificName; + + private Long deptId; + + private int checked; + + private int count; + + private String habitat; + + private double confidence; + + private Long mediaId; + + private String mediaUrl; + + private String metaUrl; + + private int mediaType; + + private int recordType; + + private Integer favorite; + + private String birdIcon; + + private int isDeleted; + + public String getRecordTime() { + if (recordTime1 != null) { +// LocalDateTime localDateTime = +// ZonedDateTime zonedDateTime = recordTime1.atZone(ZoneId.of("Asia/Shanghai")); + +// ZoneId systemDefault = ZoneId.systemDefault(); +// return zonedDateTime.format(DateTimeFormatter.ISO_DATE_TIME); +// return recordTime1.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX")); + ZonedDateTime zonedDateTime = recordTime1.atZone(ZoneId.of("Asia/Shanghai")); + + // 使用 DateTimeFormatter 格式化为 ISO 8601 格式的字符串 +// return zonedDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + ZonedDateTime utcDateTime = zonedDateTime.withZoneSameInstant(ZoneId.of("UTC")); + + // 格式化为目标字符串 + return utcDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); + + + } else { + return null; + } + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordReportData.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordReportData.java new file mode 100644 index 0000000..def9ec7 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/EsResultRecordReportData.java @@ -0,0 +1,85 @@ +package net.hzvtc.es.domain; + +import lombok.Data; +import net.hzvtc.data.check.domain.MediaItem; +import net.hzvtc.device.domain.DeviceInfo; + +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.List; + +@Data +public class EsResultRecordReportData { + private Long id; + + + private String recordTime; + + private Long deviceId; + + private DeviceInfo deviceInfo; + + private Long deptId; + + private String fullPath; + + private int checked; + + private Long mediaId; + + private String mediaUrl; + + private String metaUrl; + + private String fileName; + + private int mediaType; + +// private Integer favorite; +// +// private String memo; +// +// private String tag; +// +// private String version; + + private Long itemId; + + private List items; + + private MediaItem mediaItem; + + public void convertFrom(EsResultRecordDetails data) { + setId(data.getId()); + DateTimeFormatter fmDate = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + if (data.getRecordTime() != null) { + setRecordTime(data.getRecordTime().withZoneSameInstant(ZoneId.of("Asia/Shanghai")).format(fmDate)); + } else { + setRecordTime(""); + } + setDeptId(data.getDeptId()); + setDeviceId(data.getDeviceId()); + setMediaId(data.getMediaId()); + setMediaUrl(data.getMediaUrl()); + setMetaUrl(data.getMetaUrl()); + setMediaType(data.getMediaType()); + setItemId(data.getItemId()); + + if (mediaType == 2) { + if (metaUrl != null) { + String name = metaUrl.substring(metaUrl.lastIndexOf("/") + 1); + setFileName(name); + } + } else if (mediaType == 3) { + if (mediaUrl != null) { + String name = mediaUrl.substring(mediaUrl.lastIndexOf("/") + 1); + setFileName(name); + } + } +// setFavorite(data.getFavorite()); +// setMemo(data.getMemo()); +// setTag(data.getTag()); +// setVersion(data.getVersion()); + + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/SpeciesItem.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/SpeciesItem.java new file mode 100644 index 0000000..db4339b --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/domain/SpeciesItem.java @@ -0,0 +1,18 @@ +package net.hzvtc.es.domain; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SpeciesItem { + private Long detailsId; + private Long recordId; + private Long speciesId; + private String speciesName; + private String scientificName; + private int count; + private int isValid; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/es/mapper/EsResultRecordDetailsMapper.java b/ai-xuebao-es/src/main/java/net/hzvtc/es/mapper/EsResultRecordDetailsMapper.java new file mode 100644 index 0000000..216d193 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/es/mapper/EsResultRecordDetailsMapper.java @@ -0,0 +1,15 @@ +package net.hzvtc.es.mapper; + +import net.hzvtc.echart.domain.vo.BirdStatDataVo; +import net.hzvtc.es.domain.EsResultRecordDetailsVo; + +import java.util.List; +import java.util.Map; + +public interface EsResultRecordDetailsMapper { + List getResultRecordListForElastic(Map param); + + List getDeviceListByParam(Map map); + + List getAreaBirdSpeciesByParam(Map map); +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/service/EsRecordReportService.java b/ai-xuebao-es/src/main/java/net/hzvtc/service/EsRecordReportService.java new file mode 100644 index 0000000..45d0f50 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/service/EsRecordReportService.java @@ -0,0 +1,55 @@ +package net.hzvtc.service; + +import com.alibaba.fastjson.JSONObject; + +import java.util.Map; + +public interface EsRecordReportService { + JSONObject getBirdDailyReportStatByParam(JSONObject param); + +// String getBirdDataStatFileByParam(JSONObject param); + + String exportResultRecordListFileByParam(Map param); + + String exportValidResultRecordListFileByParam(Map param); + +// boolean sendBirdDataStatEmail(JSONObject param); + + JSONObject getSpeciesWeeklyReportByParam(JSONObject param); + + JSONObject getSpeciesMediaListByParam(JSONObject param); + + JSONObject getResultRecordStatReportByParam(JSONObject param); + + JSONObject getResultRecordValidStatReportByParam(JSONObject param); + + JSONObject getHeatValidStatReportByParam(JSONObject param); + + JSONObject getCameraRecordStatReportByParam(JSONObject param); + + JSONObject getAreaStatReportByParam(JSONObject param); + + JSONObject getSpeciesTreeList(); + + JSONObject getSpeciesMultiStatReportByParam(JSONObject param); + + JSONObject getRecordDailyStatReportByParam(JSONObject param); + + JSONObject getRecordStatDateReportByParam(JSONObject param); + + JSONObject getValidStatDateReportByParam(JSONObject param); + + JSONObject getValidRAIReportByParam(JSONObject param); + + JSONObject getValidRNReportByParam(JSONObject param); + + JSONObject getValidPercentReportByParam(JSONObject param); + + JSONObject getValidRAIRNPtReportByParam(JSONObject param); + + JSONObject getRhythmReportByParam(JSONObject param); + + JSONObject getBirdPeriodStatByParam(JSONObject object); + + JSONObject getSpeciesHabitatStatReportByParam(JSONObject param); +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/service/EsResultRecordDetailsService.java b/ai-xuebao-es/src/main/java/net/hzvtc/service/EsResultRecordDetailsService.java new file mode 100644 index 0000000..654f45a --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/service/EsResultRecordDetailsService.java @@ -0,0 +1,64 @@ +package net.hzvtc.service; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import net.hzvtc.common.core.page.TableDataInfo; +import net.hzvtc.device.domain.DeviceInfo; +import net.hzvtc.es.domain.EsResultRecordDetails; + +import java.util.List; + +public interface EsResultRecordDetailsService { + TableDataInfo querySpeciesRecordListByParam(JSONObject param, int type); + + TableDataInfo getProtectedAreaSpeciesByQuery(JSONObject param); + + TableDataInfo getResultMediaListByParam(JSONObject jsonObject); + + int deleteDocumentById(String id); + + void bulkAddResultRecordDocument(); + + int getMaxResultRecordId(); + + boolean updateResultRecordInfoById(JSONObject object); + + boolean updateResultRecordInfoByIds(JSONObject object); + + JSONObject getBirdAudioDailyStatListByParam(JSONObject param); + + JSONObject getBirdDailyStatListByParam(JSONObject param); + + JSONObject getBirdAudioMonthStatListByParam(JSONObject param); + + JSONObject getBirdAudioYearStatListByParam(JSONObject param); + + List getDeviceInfoListForPage(DeviceInfo deviceInfo); + + JSONArray getSpeciesListForSelect(); + + JSONObject getResultMediaStatByParam(); + + JSONObject getSpeciesStatListByParam(JSONObject param); + + List getDeviceInfoListForGisMap(); + + JSONObject getSpeciesStatInfo(JSONObject param); + + boolean updateSpeciesInfo(JSONObject object); + + + boolean deleteSpeciesInfo(JSONObject object); + + Long addSpeciesInfo(JSONObject object); + + List getFavoriteMediaList(JSONObject param); + + List getMediaListByItemId(JSONObject param); + +// JSONObject getBirdDailyReportStatByParam(JSONObject param); + +// String getBirdDataStatFileByParam(JSONObject param); + +// boolean sendBirdDataStatEmail(JSONObject param); +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/CommonEsUtils.java b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/CommonEsUtils.java new file mode 100644 index 0000000..30b92f8 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/CommonEsUtils.java @@ -0,0 +1,311 @@ +package net.hzvtc.service.impl; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.*; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.TrackHits; +import co.elastic.clients.json.JsonData; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import net.hzvtc.es.domain.EsResultRecordDetails; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class CommonEsUtils { + public static JSONArray getSpeciesAggregateList(JSONObject param, String indexName, ElasticsearchClient elasticsearchClient, CalendarInterval calendarInterval) throws IOException { + BoolQuery.Builder boolQueryBuilder = createQuery(param); +// log.info("========111========{}", param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0).query(q -> q.bool(boolQueryBuilder.build())).aggregations("by_speciesId", a -> a.terms(TermsAggregation.of(ta -> ta.field("species_id").size(10000))).aggregations("by_calendarInterval", b -> b.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time").calendarInterval(calendarInterval))).aggregations("sum_count_field", s -> s.sum(SumAggregation.of(sa -> sa.field("count"))))))); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate bySpeciesAgg = aggs.get("by_speciesId"); + JSONArray jsonArray = new JSONArray(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + + bySpeciesAgg.lterms().buckets().array().forEach(speciesBucket -> { + Aggregate byCalendarInterval = speciesBucket.aggregations().get("by_calendarInterval"); + byCalendarInterval.dateHistogram().buckets().array().forEach(hourBucket -> { + if (hourBucket.keyAsString() != null) { + JSONObject item = new JSONObject(); + LocalDateTime dateTime = LocalDateTime.parse(hourBucket.keyAsString(), formatter); + if (calendarInterval == CalendarInterval.Hour) { + item.put("hour", dateTime.getHour()); + } else if (calendarInterval == CalendarInterval.Day) { + item.put("weekDay", dateTime.getDayOfWeek()); + } + + item.put("speciesId", speciesBucket.key()); + Aggregate sumAgg = hourBucket.aggregations().get("sum_count_field"); + item.put("count", (int) sumAgg.sum().value()); + jsonArray.add(item); + } + }); + }); + return jsonArray; + } + + public static EsResultRecordDetails getResultRecordDetailsByParam(JSONObject param, String indexName, ElasticsearchClient elasticsearchClient) throws IOException { + BoolQuery.Builder boolQueryBuilder = createQuery(param); + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build())// + .index(indexName).query(q -> q.bool(boolQueryBuilder.build()))// + .sort(sort -> sort.field(f -> f.field("favorite").order(SortOrder.Desc)))// + .sort(sort -> sort.field(f -> f.field("id").order(SortOrder.Desc)))// +// .source(sc -> sc +// .filter(f -> f +// .includes(Arrays.asList(sources)) +// ) +// ) + .from(0).size(1), EsResultRecordDetails.class); + + List> hitList = response.hits().hits(); + + for (Hit mapHit : hitList) { + return mapHit.source(); + } + + return null; + } + + public static long getSpeciesStatAmountByParam(JSONObject param, String indexName, ElasticsearchClient elasticsearchClient) throws IOException { + BoolQuery.Builder boolQueryBuilder = createQuery(param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0).query(q -> q.bool(boolQueryBuilder.build())).aggregations("sum_count_field", a -> a.sum(SumAggregation.of(ta -> ta.field("count"))))); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate sumAgg = aggs.get("sum_count_field"); + return (long) sumAgg.sum().value(); + } + + + public static BoolQuery.Builder createQuery(JSONObject param) { +// log.info("==================param==========={}", param); + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + +// Long areaId = param.getLong("areaId"); +// if (areaId != null) { +// boolQueryBuilder.must(m -> m.match(t -> t.field("area_id").query(areaId))); +// } + +// + + Long speciesId = param.getLong("speciesId"); + if (speciesId != null) { + boolQueryBuilder.must(m -> m.nested(n -> n.path("items").query(q -> q.match(t -> t.field("items.species_id").query(speciesId))))); +// boolQueryBuilder.must(m -> m.match(t -> t.field("items.species_id").query(speciesId))); + } + + Long species_id = param.getLong("species_id"); + if (species_id != null) { +// boolQueryBuilder.must(m -> m.nested(n -> n.path("items").query(q -> q.match(t -> t.field("items.species_id").query(speciesId))))); + boolQueryBuilder.must(m -> m.match(t -> t.field("species_id").query(species_id))); + } + + Long deviceId = param.getLong("deviceId"); + if (deviceId != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("device_id").query(deviceId))); + } + + Long mediaId = param.getLong("mediaId"); + if (mediaId != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("media_id").query(mediaId))); + } + + Integer deviceType = param.getInteger("deviceType"); + if (deviceType != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("device_type").query(deviceType))); + } + + Integer favorite = param.getInteger("favorite"); + if (favorite != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("favorite").query(favorite))); + } + + Integer checked = param.getInteger("checked"); + if (checked != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("checked").query(checked))); + } + + String version = param.getString("version"); + if (version != null && !version.isEmpty()) { + boolQueryBuilder.must(m -> m.match(t -> t.field("version").query(version))); + } + Long itemId = param.getLong("itemId"); + if (itemId != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("item_id").query(itemId))); + } + + + String tag = param.getString("tag"); + if (tag != null && !tag.isEmpty()) { +// log.info("============tag==========={}", tag); +// boolQueryBuilder.must(m -> m.fuzzy(fz -> fz.field("tag").value(tag).fuzziness("1"))); + boolQueryBuilder.must(m -> m.matchPhrase(fz -> fz.field("tag").query(tag))); + } + + + String memo = param.getString("memo"); + if (memo != null && !memo.isEmpty()) { +// log.info("============memo==========={}", memo); +// boolQueryBuilder.must(m -> m.fuzzy(fz -> fz.field("memo").value(memo).fuzziness("1"))); + boolQueryBuilder.must(m -> m.matchPhrase(fz -> fz.field("memo").query(memo))); + } + + String queryStr = param.getString("queryStr"); + if (queryStr != null && !queryStr.trim().isEmpty()) { + boolQueryBuilder.should(s1 -> s1.match(fz -> fz.field("tag").query(queryStr))).should(s2 -> s2.match(fz -> fz.field("memo").query(queryStr))).minimumShouldMatch("1"); + } + + Integer isDeleted = param.getInteger("isDeleted"); + if (isDeleted != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("is_deleted").query(isDeleted))); + } + + String dateFrom = param.getString("dateFrom"); + if (dateFrom != null && !dateFrom.isEmpty()) { + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").gte(JsonData.of(dateFrom)).timeZone("+08:00"))); + } + + String dateTo = param.getString("dateTo"); + if (dateTo != null && !dateTo.isEmpty()) { + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").lte(JsonData.of(dateTo)).timeZone("+08:00"))); + } + + String startYear = param.getString("startYear"); + if (startYear != null && !startYear.isEmpty()) { + startYear = startYear + "-01-01"; + String finalStartYear = startYear; + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").gte(JsonData.of(finalStartYear)).timeZone("+08:00"))); + } + + String endYear = param.getString("endYear"); + if (endYear != null && !endYear.isEmpty()) { + endYear = endYear + "-12-31"; + String finalEndYear = endYear; + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").lte(JsonData.of(finalEndYear)))); + } + + Integer recordType = param.getInteger("recordType"); + if (recordType != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("record_type").query(recordType))); + } + + String calcDate = param.getString("calcDate"); + if (calcDate != null && !calcDate.isEmpty()) { + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").lte(JsonData.of(calcDate)).gte(JsonData.of(calcDate)).timeZone("+08:00"))); + } + + String calcYear = param.getString("calcYear"); + if (calcYear != null && !calcYear.isEmpty()) { + String from = calcYear + "-01-01"; + String to = calcYear + "-12-31"; + boolQueryBuilder.must(m -> m.range(v -> v.field("record_time").format("yyyy-MM-dd").lte(JsonData.of(to)).gte(JsonData.of(from)).timeZone("+08:00"))); + } + + JSONArray array = param.getJSONArray("exSpeciesIds"); + if (array != null && !array.isEmpty()) { +// log.info("==========exSpeciesIds=========={}", array); + createExcludeArrayQuery("species_id", null, array, boolQueryBuilder); + } + + array = param.getJSONArray("ids"); + if (array != null && !array.isEmpty()) { + createExcludeArrayQuery("items", "items.species_id", array, boolQueryBuilder); + } + + array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + createIncludeNestArrayQuery("items", "items.species_id", array, boolQueryBuilder); + } + + array = param.getJSONArray("speciesInIds"); + if (array != null && !array.isEmpty()) { + createIncludeArrayQuery("species_id", array, boolQueryBuilder); + } + + Integer mediaType = param.getInteger("mediaType"); + if (mediaType != null) { + boolQueryBuilder.must(m -> m.match(t -> t.field("media_type").query(mediaType))); + } + + String mediaTypes = param.getString("mediaTypes"); + if (mediaTypes != null && !mediaTypes.isEmpty()) { + String[] arr = mediaTypes.split(","); + List terms = Arrays.asList(arr); + List fieldValues = terms.stream().map(FieldValue::of).collect(Collectors.toList()); + boolQueryBuilder.must(m -> m.terms(t -> t.field("media_type").terms(tq -> tq.value(fieldValues)))); + } + + array = param.getJSONArray("deviceIds"); +// log.info("=============device ids==={}", array); + createIncludeArrayQuery("device_id", array, boolQueryBuilder); + + array = param.getJSONArray("inDocIds"); + createIncludeArrayQuery("id", array, boolQueryBuilder); +// if (array != null && !array.isEmpty()) { +// List list = new ArrayList<>(); +// for (Object id : array) { +// list.add((Long) id); +// } +// List fieldValues = list.stream().map(FieldValue::of).collect(Collectors.toList()); +// boolQueryBuilder.mustNot(m -> m.terms(t -> t.field("device_id").terms(tq -> tq.value(fieldValues)))); +// } +// log.info("============={}", boolQueryBuilder); + return boolQueryBuilder; + } + + private static void createIncludeArrayQuery(String fieldName, JSONArray array, BoolQuery.Builder boolQueryBuilder) { + if (array != null && !array.isEmpty()) { + List list = new ArrayList<>(); + for (Object id : array) { + list.add(Long.parseLong(id.toString())); + } + List fieldValues = list.stream().map(FieldValue::of).collect(Collectors.toList()); + boolQueryBuilder.must(m -> m.terms(t -> t.field(fieldName).terms(tq -> tq.value(fieldValues)))); + } + } + + private static void createIncludeNestArrayQuery(String fieldName, String nestedName, JSONArray array, BoolQuery.Builder boolQueryBuilder) { + if (array != null && !array.isEmpty()) { + List list = new ArrayList<>(); + for (Object id : array) { + list.add(Long.parseLong(id.toString())); + } + List fieldValues = list.stream().map(FieldValue::of).collect(Collectors.toList()); + if (nestedName != null && !nestedName.isEmpty()) { + boolQueryBuilder.must(mn -> mn.nested(n -> n.path(fieldName).query(nq -> nq.terms(t -> t.field(nestedName).terms(tq -> tq.value(fieldValues)))))); + } else { + boolQueryBuilder.must(m -> m.terms(t -> t.field(fieldName).terms(tq -> tq.value(fieldValues)))); + } + + } + } + + private static void createExcludeArrayQuery(String fieldName, String nestedName, JSONArray array, BoolQuery.Builder boolQueryBuilder) { + if (array != null && !array.isEmpty()) { + List list = new ArrayList<>(); + for (Object id : array) { + list.add((Long) id); + } + List fieldValues = list.stream().map(FieldValue::of).collect(Collectors.toList()); + if (nestedName != null && !nestedName.isEmpty()) { + boolQueryBuilder.mustNot(mn -> mn.nested(n -> n.path(fieldName).query(nq -> nq.terms(t -> t.field(nestedName).terms(tq -> tq.value(fieldValues)))))); + } else { + boolQueryBuilder.mustNot(m -> m.terms(t -> t.field(fieldName).terms(tq -> tq.value(fieldValues)))); + } + + } + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsRecordReportServiceImpl.java b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsRecordReportServiceImpl.java new file mode 100644 index 0000000..d0ca3b4 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsRecordReportServiceImpl.java @@ -0,0 +1,2594 @@ +package net.hzvtc.service.impl; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.Time; +import co.elastic.clients.elasticsearch._types.aggregations.*; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.TrackHits; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.hzvtc.area.service.ProtectedAreaService; +import net.hzvtc.area.service.ProtectedAreaSpeciesService; +import net.hzvtc.bird.service.BirdSpeciesService; +import net.hzvtc.common.core.domain.entity.SysDept; +import net.hzvtc.data.check.service.MediaItemService; +import net.hzvtc.device.domain.DeviceInfo; +import net.hzvtc.device.service.DeviceInfoService; +import net.hzvtc.echart.domain.vo.BirdStatDataVo; +import net.hzvtc.email.service.EmailService; +import net.hzvtc.es.domain.EsResultRecordDetails; +import net.hzvtc.es.domain.EsResultRecordReportData; +import net.hzvtc.es.domain.SpeciesItem; +import net.hzvtc.es.mapper.EsResultRecordDetailsMapper; +import net.hzvtc.service.EsRecordReportService; +import net.hzvtc.system.mapper.SysDeptMapper; +import net.hzvtc.web.stat.service.SpeciesDailyStatService; +import org.apache.poi.ss.util.CellRangeAddress; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.temporal.TemporalAdjusters; +import java.util.*; +import java.util.stream.Collectors; + + +@Service +@RequiredArgsConstructor(onConstructor = @__(@Lazy)) +@Slf4j +public class EsRecordReportServiceImpl implements EsRecordReportService { + private static final Logger logger = LoggerFactory.getLogger(EsRecordReportService.class); + @Resource + private final EsResultRecordDetailsMapper esResultRecordDetailsMapper; + + private final ElasticsearchClient elasticsearchClient; + + private final EmailService emailService; + + private final SpeciesDailyStatService speciesDailyStatService; + + private final ProtectedAreaSpeciesService protectedAreaSpeciesService; + + private final BirdSpeciesService birdSpeciesService; + + private final DeviceInfoService deviceInfoService; + + private final SysDeptMapper sysDeptMapper; + + private final MediaItemService mediaItemService; + + @Value("${hzvtc.dailyReportFilePath}") + private String dailyReportFilePath; + + @Value("${hzvtc.dailyReportFileOutPath}") + private String dailyReportFileOutPath; + + private final ProtectedAreaService protectedAreaService; + + private final String indexName = "animal_record_details"; + + private final String validIndexName = "animal_valid_record"; + + @Override + public JSONObject getBirdDailyReportStatByParam(JSONObject param) { + String calcDate = param.getString("calcDate"); + if (calcDate == null || calcDate.isEmpty()) { + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + param.put("calcDate", df.format(new Date())); + } + + JSONObject result = new JSONObject(); + JSONArray deviceData = getSpeciesDailyAmountByDeviceId(param); + Object[] array = compactJSONArray(deviceData); + result.put("deviceData", deviceData); + Map map = createQueryMap(param); + + List list = esResultRecordDetailsMapper.getAreaBirdSpeciesByParam(map); + JSONArray array1 = new JSONArray(); + JSONArray array2 = new JSONArray(); + Long areaId = (Long) map.get("areaId"); + param.clear(); + param.put("areaId", areaId); + param.put("calcDate", calcDate); + if (list != null && !list.isEmpty()) { + list.forEach(item -> { + JSONObject obj = new JSONObject(); + Long id = item.getId(); + obj.put("id", id); + obj.put("name", item.getName()); + obj.put("list", filterArrayBySpeciesId(array, id)); + array1.add(obj); + try { + JSONObject obj1 = new JSONObject(); + param.put("speciesId", item.getId()); + obj1.put("id", id); + obj1.put("name", item.getName()); + param.put("recordType", 1); +// log.info("=================={}", param); + JSONArray arr = getSpeciesHabitatAggregateList(param); + if (arr.isEmpty()) { + param.put("recordType", 2); + arr = getSpeciesHabitatAggregateList(param); + } + obj1.put("list", arr); + array2.add(obj1); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + }); + } + result.put("speciesData", array1); + result.put("habitatData", array2); + result.put("mediaList", new ArrayList<>()); + return result; + } + +// @Override +// public String getBirdDataStatFileByParam(JSONObject param) { +// try { +// Long areaId = param.getLong("areaId"); +// String areaName = ""; +// +// if (areaId != null) { +// ProtectedArea protectedArea = protectedAreaService.selectProtectedAreaById(areaId); +// areaName = protectedArea.getName(); +// } else { +// return "500"; +// } +// XSSFWorkbook xssfWorkbook = new XSSFWorkbook(Files.newInputStream(Paths.get(dailyReportFilePath + "daily_report_tmpl.xlsx"))); +// XSSFSheet sheet = xssfWorkbook.getSheetAt(0); +// sheet.getRow(1).getCell(1).setCellValue(areaName + "鸟类监测日报"); +// String calcDate = param.getString("calcDate"); +// if (calcDate == null || calcDate.isEmpty()) { +// SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); +// calcDate = df.format(new Date()); +// } +// SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss"); +// sheet.getRow(2).getCell(1).setCellValue("日期:" + calcDate); +// sheet.getRow(4).getCell(1).setCellValue(areaName + "每日鸟类智慧监测数据,包含鸟类监测总数,关键鸟类生境分布和关键物种分时监测数量"); +// JSONObject result = getBirdDailyReportStatByParam(param); +// // int baseDiffRow = 0; +// JSONArray array = result.getJSONArray("deviceData"); +// int[] sum = {0, 0, 0}; +// if (array != null) { +// AtomicInteger starRow = new AtomicInteger(8); +// +// array.forEach(data -> { +// JSONObject obj = (JSONObject) data; +// int cell = 1; +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(obj.getString("name")); +// JSONArray subarray = obj.getJSONArray("list"); +// int[] value = {0, 0, 0}; +// if (subarray != null) { +// subarray.stream().map(el -> (JSONObject) el).forEach(element -> { +// int hour = element.getInteger("hour"); +// int count = element.getInteger("count"); +// if (hour <= 12) { +// value[0] += count; +// } else { +// value[1] += count; +// } +// value[2] = Math.max(value[0], value[1]); +// }); +// } +// +// for (int i = 0; i < sum.length; i++) { +// sum[i] += value[i]; +// } +// for (int i = 0; i < value.length; i++) { +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(value[i]); +// } +// +// starRow.getAndIncrement(); +// }); +// int cell = 2; +// for (int i = 0; i < sum.length; i++) { +// sheet.getRow(28).getCell(cell++).setCellValue(sum[i]); +// } +// for (int i = 0; i < 20 - array.size(); i++) { +// XSSFRow row = sheet.getRow(8 + array.size() + i); +// row.setZeroHeight(true); +// } +// } +// +// array = result.getJSONArray("habitatData"); +// if (array != null) { +// AtomicInteger starRow = new AtomicInteger(32); +// array.forEach(data -> { +// JSONObject obj = (JSONObject) data; +// int cell = 1; +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(obj.getString("name")); +// int[] value = {0, 0, 0, 0, 0}; +// +// JSONArray subArray = obj.getJSONArray("list"); +// if (subArray != null) { +// subArray.stream().map(el -> (JSONObject) el).forEach(element -> { +// String code = element.getString("habitat"); +// int count = element.getInteger("count"); +// if ("ground".equals(code)) { +// value[0] = count; +// } else if ("shoreside".equals(code)) { +// value[1] = count; +// } else if ("water".equals(code)) { +// value[2] = count; +// } else if ("tree".equals(code)) { +// value[3] = count; +// } else { +// value[4] = count; +// } +// }); +// } +// for (int i = 0; i < value.length; i++) { +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(value[i]); +// } +// +// starRow.getAndIncrement(); +// }); +// } +// +// +// array = result.getJSONArray("speciesData"); +// if (array != null) { +// AtomicInteger starRow = new AtomicInteger(43); +// List list; +// if (array.size() > 8) { +// list = array.subList(0, 8); +// } else { +// list = array.subList(0, array.size()); +// } +// list.forEach(data -> { +// JSONObject obj = (JSONObject) data; +// int cell = 1; +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(obj.getString("name")); +// int[] value = new int[11]; +// JSONArray subArray = obj.getJSONArray("list"); +// final int[] index = {0}; +// if (subArray != null) { +// subArray.forEach(el -> { +// if (index[0] >= 0 && index[0] < 11) { +// value[index[0]] = (int) el; +// } +// index[0]++; +// }); +// } +// +// for (int i = 0; i < value.length; i++) { +// sheet.getRow(starRow.get()).getCell(cell++).setCellValue(value[i]); +// } +// //int max = Collections.max(Arrays.asList(value)); +// Arrays.sort(value); +// int max = value[value.length - 1]; +// XSSFRow row = sheet.getRow(starRow.get()); +// if (row == null) { +// row = sheet.createRow(starRow.get()); +// } +// XSSFCell xssfCell = row.getCell(cell); +// if (xssfCell == null) { +// xssfCell = row.createCell(cell); +// } +// xssfCell.setCellValue(max); +// cell++; +// starRow.getAndIncrement(); +// }); +// } +// +// String outfile = dailyReportFileOutPath + areaName + "鸟类监测日报(日期" + calcDate + ")" + ".xlsx"; +// FileOutputStream fileOut = new FileOutputStream(outfile); +// xssfWorkbook.write(fileOut); +// xssfWorkbook.close(); +// fileOut.close(); +//// emailService.sendEmailWithAttachment("这个是一份带附件的测试邮件", "附件为文件,请查收", outfile, "935195927@qq.com"); +// return outfile; +// } catch (Exception ex) { +// logger.error(ex.getMessage()); + + /// / ex.printStackTrace(); +// } +// return null; +// } + private void createTableRow(int start, int count, XSSFSheet sheet, XSSFRow firstRow) { + for (int j = start; j < start + count; j++) { + sheet.createRow(j); + XSSFRow currentRow = sheet.getRow(j); + for (int i = 0; i < firstRow.getLastCellNum(); i++) { + XSSFCell prevCell = firstRow.getCell(i); + if (prevCell != null) { + // 创建当前行的单元格(如果不存在) + XSSFCell newCell = currentRow.getCell(i); + if (newCell == null) { + newCell = currentRow.createCell(i); + } + newCell.setCellStyle(prevCell.getCellStyle()); + + + } + } + } + + } + + @Override + public String exportResultRecordListFileByParam(Map param) { + param.remove("speciesInIds"); + String str = (String) param.get("dataType"); + int dataType = 1; + if (str != null && !str.isEmpty()) { + dataType = Integer.parseInt(str); + } + JSONObject obj = new JSONObject(); + for (String key : param.keySet()) { + obj.put(key, param.get(key)); + } + try { + if (!getDeviceIdsByBoundBox(obj)) { + return null; + } + List list = getResultRecordListByParam(obj); + if (list == null) { + return null; + } + XSSFWorkbook xssfWorkbook = null; + if (dataType == 1) { + xssfWorkbook = new XSSFWorkbook(Files.newInputStream(Paths.get(dailyReportFilePath + "template01.xlsx"))); + } else { + xssfWorkbook = new XSSFWorkbook(Files.newInputStream(Paths.get(dailyReportFilePath + "template02.xlsx"))); + } + + int startRow = 2; + XSSFSheet sheet = xssfWorkbook.getSheetAt(0); + XSSFRow firstRow = sheet.getRow(startRow - 1); + int cell = 0; + int index = 1; + for (EsResultRecordReportData data : list) { + createTableRow(startRow, data.getItems().size(), sheet, firstRow); + cell = 0; + if (data.getItems().size() > 1) { + for (int i = 0; i < 20; i++) { + sheet.addMergedRegion(new CellRangeAddress(startRow, startRow + data.getItems().size() - 1, i, i)); + } + } + sheet.getRow(startRow).getCell(cell++).setCellValue(index++); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getId()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getId()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getFileName()); + str = data.getDeviceInfo() == null ? "" : data.getDeviceInfo().getDeptFullName(); + if (str != null && !str.isEmpty()) { + String[] array = str.split(","); + int i = 0; + for (String name : array) { + sheet.getRow(startRow).getCell(cell++).setCellValue(array[i++]); + } + for (int j = 0; j < 3 - array.length; j++) { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + } else { + for (int j = 0; j < 3; j++) { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + } + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getPid()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getPid()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getDeviceIp()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getRecordTime()); + if (data.getMediaItem() != null) { + if (dataType == 1) { + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getCameraVolt()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getMemory()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getNstatus()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getOntime()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getAstatus()); + } + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getLon()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getLat()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getElevation()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getTemperature()); + } else { + if (dataType == 1) { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } +// log.info("=====cell={}", cell); + for (SpeciesItem speciesItem : data.getItems()) { + int icol = cell; +// log.info("=====icol={}", icol); + sheet.getRow(startRow).getCell(icol++).setCellValue(speciesItem.getSpeciesName()); + sheet.getRow(startRow).getCell(icol++).setCellValue(speciesItem.getCount()); + if (dataType == 2) { + sheet.getRow(startRow++).getCell(icol).setCellValue(speciesItem.getIsValid()); + } else { + startRow++; + } + } + } + String outfile = ""; + if (dataType == 0) { + outfile = dailyReportFileOutPath + "红外相机照片_视频数据记录表" + ".xlsx"; + } else { + outfile = dailyReportFileOutPath + "红外相机独立有效数据表" + ".xlsx"; + } + FileOutputStream fileOut = new FileOutputStream(outfile); + xssfWorkbook.write(fileOut); + xssfWorkbook.close(); + fileOut.close(); + return outfile; + } catch (Exception ex) { +// ex.printStackTrace(); + log.error("error", ex); + } + return null; + } + + @Override + public String exportValidResultRecordListFileByParam(Map param) { + JSONObject obj = new JSONObject(); + for (String key : param.keySet()) { + obj.put(key, param.get(key)); + } + try { + if (!getDeviceIdsByBoundBox(obj)) { + return null; + } + List list = getResultRecordListByParam(obj); + if (list == null) { + return null; + } + XSSFWorkbook xssfWorkbook = new XSSFWorkbook(Files.newInputStream(Paths.get(dailyReportFilePath + "template01.xlsx"))); + int startRow = 2; + XSSFSheet sheet = xssfWorkbook.getSheetAt(0); + XSSFRow firstRow = sheet.getRow(startRow - 1); + int cell = 0; + int index = 1; + for (EsResultRecordReportData data : list) { + createTableRow(startRow, data.getItems().size(), sheet, firstRow); + cell = 0; + if (data.getItems().size() > 1) { + for (int i = 0; i < 20; i++) { + sheet.addMergedRegion(new CellRangeAddress(startRow, startRow + data.getItems().size() - 1, i, i)); + } + } + sheet.getRow(startRow).getCell(cell++).setCellValue(index++); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getId()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getId()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getFileName()); + String str = data.getDeviceInfo() == null ? "" : data.getDeviceInfo().getDeptFullName(); + if (str != null && !str.isEmpty()) { + String[] array = str.split(","); + int i = 0; + for (String name : array) { + sheet.getRow(startRow).getCell(cell++).setCellValue(array[i++]); + } + for (int j = 0; j < 3 - array.length; j++) { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + } else { + for (int j = 0; j < 3; j++) { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + } + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getPid()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getDuid()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getDeviceInfo().getDeviceIp()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getRecordTime()); + if (data.getMediaItem() != null) { + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getCameraVolt()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getMemory()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getNstatus()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getOntime()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getAstatus()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getLon()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getLat()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getElevation()); + sheet.getRow(startRow).getCell(cell++).setCellValue(data.getMediaItem().getTemperature()); + } else { + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + sheet.getRow(startRow).getCell(cell++).setCellValue(""); + } + for (SpeciesItem speciesItem : data.getItems()) { + int icol = cell; + sheet.getRow(startRow).getCell(icol++).setCellValue(speciesItem.getSpeciesName()); + sheet.getRow(startRow++).getCell(icol++).setCellValue(speciesItem.getCount()); + } + } + String outfile = dailyReportFileOutPath + "红外相机照片_视频数据记录表" + ".xlsx"; + FileOutputStream fileOut = new FileOutputStream(outfile); + xssfWorkbook.write(fileOut); + xssfWorkbook.close(); + fileOut.close(); + return outfile; + } catch (Exception ex) { +// ex.printStackTrace(); + log.error("error", ex); + } + return null; + } + + private String convertNull(String str) { + return str == null ? "" : str; + } + + private boolean isValidSpeciesId(Long speciesId, JSONArray array) { + if (speciesId == 0L) { + return false; + } else { + if (array != null && !array.isEmpty()) { + return array.contains(speciesId); + } else { + return true; + } + } + } + + private List getResultRecordListByParam(JSONObject param) { + param.put("is_deleted", 0); + String speciesIds = param.getString("speciesIds"); + JSONArray jsonArray = null; + if (speciesIds != null && !speciesIds.isEmpty()) { + jsonArray = new JSONArray(); + String[] array = speciesIds.split(","); + for (String str : array) { + jsonArray.add(Long.parseLong(str)); + } + param.put("speciesIds", jsonArray); + } + String excludeIds = param.getString("excludeIds"); + if (excludeIds != null && !excludeIds.isEmpty()) { + String[] array = excludeIds.split("_"); + JSONArray ids = new JSONArray(); + for (String str : array) { + ids.add(Long.parseLong(str)); + } + if (!ids.isEmpty()) { + param.put("exSpeciesIds", ids); + } + } + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + try { + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .scroll(t -> t.time("1m")).size(1000).query(q -> q.bool(boolQueryBuilder.build())).sort(sort -> sort.field(f -> f.field("record_time").order(SortOrder.Asc))), EsResultRecordDetails.class); + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + Map map = new HashMap<>(); + for (Hit mapHit : hitList) { + map.clear(); + EsResultRecordDetails obj = mapHit.source(); + if (obj != null) { + EsResultRecordReportData data = new EsResultRecordReportData(); + data.convertFrom(obj); + data.setDeviceInfo(deviceInfoService.getDeviceInfoForElasticById(obj.getDeviceId())); + + for (SpeciesItem item : obj.getItems()) { + if (isValidSpeciesId(item.getSpeciesId(), jsonArray)) { + SpeciesItem speciesItem = map.get(item.getSpeciesId()); + if (speciesItem != null) { + item.setCount(speciesItem.getCount() + item.getCount()); + } + map.put(item.getSpeciesId(), item); + } + } + if (!map.isEmpty()) { + data.setItems(new ArrayList<>(map.values())); + data.setMediaItem(mediaItemService.selectMediaItemById(obj.getItemId())); + list.add(data); + } + } + } + + return list; + } catch (Exception ex) { + logger.error(ex.getMessage()); + } + return null; + } + + +// @Override +// public boolean sendBirdDataStatEmail(JSONObject param) { +// String areaId = param.getString("areaId"); +// String calcDate = param.getString("calcDate"); +// if (calcDate == null || calcDate.isEmpty()) { +// SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); +// calcDate = df.format(new Date()); +// } +// ProtectedArea data = null; +// if (areaId != null && !areaId.isEmpty()) { +// data = protectedAreaService.selectProtectedAreaById(Long.parseLong(areaId)); +// +// } else { +// return false; +// } +// +// String filePath = getBirdDataStatFileByParam(param); +// +// if (filePath != null && !filePath.isEmpty()) { +// emailService.sendEmailWithAttachment(data.getName() + "鸟类监测日报,检测时间:" + calcDate, "附件为鸟类监测日报,请查收", filePath, data.getEmail()); +// return true; +// } else { +// return false; +// } +// } + + @Override + public JSONObject getSpeciesWeeklyReportByParam(JSONObject param) { + + return null; + } + + private Map createQueryMap(JSONObject param) { + Map map = new HashMap<>(); + Long areaId = param.getLong("areaId"); + map.put("areaId", areaId); + + String isCommon = param.getString("isCommon"); + if (isCommon != null && !isCommon.isEmpty()) { + map.put("isCommon", Integer.parseInt(isCommon)); + } else { + map.put("isCommon", 1); + } + map.put("limit", 8); + return map; + } + + @Override + public JSONObject getSpeciesMediaListByParam(JSONObject param) { + JSONObject result = new JSONObject(); + Map map = createQueryMap(param); + List list = protectedAreaSpeciesService.getProtectedAreaSpeciesListByAreaId(map); + if (list == null) { + return result; + } + JSONObject queryMap = new JSONObject(); + list.forEach(item -> { + queryMap.put("areaId", map.get("areaId")); + queryMap.put("speciesId", item.getSpeciesId()); + queryMap.put("mediaType", 3); + try { + EsResultRecordDetails data = CommonEsUtils.getResultRecordDetailsByParam(queryMap, indexName, elasticsearchClient); + if (data != null) { + item.setMediaId(data.getMediaId()); + item.setMediaUrl(data.getMediaUrl()); + item.setMetaUrl(data.getMetaUrl()); + queryMap.clear(); + queryMap.put("areaId", map.get("areaId")); + queryMap.put("speciesId", item.getSpeciesId()); + + //-----revise +// item.setTotal(speciesDailyStatService.getSpeciesDailyStatAmountByParam(queryMap)); + + LocalDate now = LocalDate.now(); + queryMap.put("curDate", now.toString()); + + //-----revise +// item.setToday(speciesDailyStatService.getSpeciesDailyStatAmountByParam(queryMap)); + queryMap.put("curDate", now.plusDays(-1).toString()); + + //-----revise +// item.setYesterday(speciesDailyStatService.getSpeciesDailyStatAmountByParam(queryMap)); + } + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + }); + result.put("list", list); + return result; + } + + @Override + public JSONObject getResultRecordStatReportByParam(JSONObject param) { +// log.info("==========1============={}", param); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + JSONObject result = new JSONObject(); +// String excludeIds = param.getString("excludeIds"); +// if (excludeIds != null && !excludeIds.isEmpty()) { +// String[] array = excludeIds.split("_"); +// JSONArray ids = new JSONArray(); +// for (String str : array) { +// ids.add(Long.parseLong(str)); +// } +// if (!ids.isEmpty()) { +// param.put("ids", ids); +// } +// } +// JSONArray array = param.getJSONArray("speciesIds"); + +// JSONObject queryParam = createRecordQueryParam(param); +// if (queryParam == null) { +// return result; +// } +// log.info("==============queryParam==={}", queryParam); +// result.put("dateFrom", queryParam.getString("dateFrom")); +// result.put("dateTo", queryParam.getString("dateTo")); +// result.put("deviceAmount", queryParam.getInteger("deviceAmount")); +// result.put("list", getResultStatListByParam(queryParam)); + result.put("list", getCameraStatListByParam(param)); + return result; + } + + + @Override + public JSONObject getResultRecordValidStatReportByParam(JSONObject param) { + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + JSONObject result = new JSONObject(); +// JSONArray array = param.getJSONArray("speciesIds"); +// if(array != null && !array.isEmpty()) { +// param.put("speciesInIds", array); +// } +// JSONObject queryParam = createRecordQueryParam(param); +// if (queryParam == null) { +// return result; +// } + +// result.put("dateFrom", queryParam.getString("dateFrom")); +// result.put("dateTo", queryParam.getString("dateTo")); + result.put("list", getResultValidAllStatListByParam(param)); + return result; + } + + @Override + public JSONObject getHeatValidStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); + if (getDeviceIdsByBoundBox(param)) { + result.put("list", getHeatValidStatListByParam(param)); + } + + return result; + } + + private List getHeatValidStatListByParam(JSONObject queryParam) { + // 处理 excludeIds 参数 +// String excludeIds = queryParam.getString("excludeIds"); +// if (excludeIds != null && !excludeIds.isEmpty()) { +// String[] array = excludeIds.split("_"); +// JSONArray ids = new JSONArray(); +// for (String str : array) { +// try { +// ids.add(Long.parseLong(str)); +// } catch (NumberFormatException e) { +// logger.error("Failed to parse excludeId: " + str, e); +// } +// } +// if (!ids.isEmpty()) { +// queryParam.put("exSpeciesIds", ids); +// } +// } + + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("device_id").size(10000) // + ))// + .aggregations("by_species_id", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("species_id").size(10000))).aggregations("sum_value",// + sumAgg -> sumAgg.sum(m -> m.field("count"))))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByDeviceId = aggregations.get("by_device_id"); + List list = new ArrayList<>(); + if (aggByDeviceId != null) { + aggByDeviceId.lterms().buckets().array().forEach(typeBucket -> { + Long deviceId = (Long) typeBucket.key(); + Map birdMap = new HashMap<>(); + Map map = new HashMap<>(); + JSONObject data = new JSONObject(); + // + DeviceInfo deviceInfo = deviceInfoService.selectDeviceInfoById(deviceId); + data.put("data", deviceInfo); + int[] amount = {0}; + LongTermsAggregate aggBySpeciesId = typeBucket.aggregations().get("by_species_id").lterms(); + if (aggBySpeciesId != null) { + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + JSONObject obj = map.get(speciesId); + int count = (int) bucket.aggregations().get("sum_value").sum().value(); + amount[0] += count; + if (obj == null) { + obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put("count", count); + } else { + int countOld = obj.getIntValue("count"); + obj.put("count", count + countOld); + } + map.put(speciesId, obj); + }); + data.put("list", new ArrayList<>(map.values())); + } + data.put("amount", amount[0]); + list.add(data); + }); + return list; + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private boolean getDeviceIdsByBoundBox(JSONObject param) { + String excludeIds = param.getString("excludeIds"); + if (excludeIds != null && !excludeIds.isEmpty()) { + String[] array = excludeIds.split("_"); + JSONArray ids = new JSONArray(); + for (String str : array) { + ids.add(Long.parseLong(str)); + } + if (!ids.isEmpty()) { + param.put("exSpeciesIds", ids); + } + } + + String boundBox = param.getString("boundBox"); + if (boundBox != null && !boundBox.isEmpty()) { + String[] array = boundBox.split(","); + if (array.length == 4) { + Map obj = new HashMap<>(); + obj.put("latFrom", Double.parseDouble(array[0])); + obj.put("latTo", Double.parseDouble(array[1])); + obj.put("lngFrom", Double.parseDouble(array[2])); + obj.put("lngTo", Double.parseDouble(array[3])); + List deviceIds = deviceInfoService.getDeviceIdsByBoundBox(obj); + if (!deviceIds.isEmpty()) { + param.put("deviceIds", deviceIds); + return true; + } else { + return false; + } + } + } + + Long deptId = param.getLong("deptId"); + if (deptId != null) { + List deptIds = sysDeptMapper.getDeptSubIdsById(deptId); + List deviceIds = deviceInfoService.getDeviceIdsInList(deptIds); + if (deviceIds == null || deviceIds.isEmpty()) { + return false; + } else { + param.put("deviceIds", deviceIds); + return true; + } + } + return true; + } + + private JSONObject createRecordQueryParam(JSONObject param) { + JSONObject queryParam = calcDateRange(param); + JSONArray ids = new JSONArray(); + ids.add(0L); + queryParam.put("ids", ids); + Long deptId = param.getLong("deptId"); + + if (deptId != null) { + List deviceIds = deviceInfoService.getDeviceIdListByDeptId(deptId); + if (!deviceIds.isEmpty()) { +// log.info("==============device ids==={}", deviceIds); + queryParam.put("deviceIds", deviceIds); + queryParam.put("deviceAmount", deviceIds.size()); + } else { + return null; + } + } else { + queryParam.put("deviceAmount", deviceInfoService.getDeviceAmountByParam()); + } + String boundBox = param.getString("boundBox"); + if (boundBox != null && !boundBox.isEmpty()) { + String[] array = boundBox.split(","); + if (array.length == 4) { + Map obj = new HashMap<>(); + obj.put("latFrom", Double.parseDouble(array[0])); + obj.put("latTo", Double.parseDouble(array[1])); + obj.put("lngFrom", Double.parseDouble(array[2])); + obj.put("lngTo", Double.parseDouble(array[3])); + List deviceIds = deviceInfoService.getDeviceIdsByBoundBox(obj); + if (!deviceIds.isEmpty()) { + queryParam.put("deviceIds", deviceIds); + } else { + return null; + } + } + } + return queryParam; + } + + @Override + public JSONObject getCameraRecordStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); +// JSONObject queryParam = calcDateRange(param); + JSONObject queryParam = createRecordQueryParam(param); + if (queryParam == null) { + return result; + } + result.put("dateFrom", queryParam.getString("dateFrom")); + result.put("dateTo", queryParam.getString("dateTo")); +// result.put("amount", getCameraAmountByParam(queryParam)); + result.put("list", getSpeciesAmountInCamera(queryParam)); + return result; + } + + @Override + public JSONObject getAreaStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); +// log.info("param==={}",param); +// JSONObject queryParam = createRecordQueryParam(param); +// log.info("queryParam==={}",queryParam); +// if (queryParam == null) { +// return result; +// } + + getDeviceIdsByBoundBox(param); + + List deviceList = getDeviceListByParam(param); + +// log.info("deviceList===={}",deviceList); + + if (deviceList != null && !deviceList.isEmpty()) { + List deptList = sysDeptMapper.getDeptAreaListByDevices(deviceList); +// log.info("deptlist===={}",deptList); + result.put("list", deptList); + } else { + result.put("list", new ArrayList<>()); + } + return result; + } + + + @Override + public JSONObject getSpeciesTreeList() { + JSONObject result = new JSONObject(); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .size(0) // +// .aggregations("items",n->n.nested("items")) + .aggregations("by_species_id", agg -> agg.terms(t -> t.field("species_id").size(1000))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + log.info("====={}", aggBySpeciesId); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(longTermsBucket -> { + JSONObject obj = new JSONObject(); + long id = longTermsBucket.key(); + obj.put("id", id); + obj.put("name", birdSpeciesService.getSpeciesNameById(id)); + array.add(obj); + }); + } + array.sort(Comparator.comparing(obj -> ((JSONObject) obj).getLong("id"))); + result.put("data", array); + return result; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + @Override + public JSONObject getSpeciesMultiStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); + JSONObject queryParam = calcDateRange(param); + result.put("dateFrom", queryParam.getString("dateFrom")); + result.put("dateTo", queryParam.getString("dateTo")); + result.put("list", getSpeciesStatByParam(queryParam)); + return result; + } + + @Override + public JSONObject getRecordDailyStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); + JSONObject queryParam = createRecordQueryParam(param); + result.put("dateFrom", queryParam.getString("dateFrom")); + result.put("dateTo", queryParam.getString("dateTo")); +// log.info("==========1======{}", queryParam); + result.put("list", getSpeciesDailyStatByParam(queryParam)); + return result; + } + + @Override + public JSONObject getRecordStatDateReportByParam(JSONObject param) { + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + JSONObject result = new JSONObject(); + Integer dateType = param.getInteger("dateType"); + if (dateType == 1) { + result.put("list", dataStatByTime(param, CalendarInterval.Year, "yyyy")); + } else if (dateType == 2) { + result.put("list", dataStatByTime(param, CalendarInterval.Quarter, "yyyy-QQQ")); + } else if (dateType == 3) { + result.put("list", dataStatByTime(param, CalendarInterval.Month, "yyyy-MM")); + } else if (dateType == 4) { + result.put("list", dataStatByTime(param, CalendarInterval.Day, "yyyy-MM-dd")); + } + return result; + } + + @Override + public JSONObject getValidStatDateReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + + JSONObject result = new JSONObject(); + Integer dateType = param.getInteger("dateType"); + if (dateType == 1) { + result.put("list", dataValidStatByTime(param, CalendarInterval.Year, "yyyy")); + } else if (dateType == 2) { + result.put("list", dataValidStatByTime(param, CalendarInterval.Quarter, "yyyy-QQQ")); + } else if (dateType == 3) { + result.put("list", dataValidStatByTime(param, CalendarInterval.Month, "yyyy-MM")); + } else if (dateType == 4) { + result.put("list", dataValidStatByTime(param, CalendarInterval.Day, "yyyy-MM-dd")); + } +// CalendarInterval.Hour + return result; + } + + @Override + public JSONObject getValidRAIReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + JSONObject result = new JSONObject(); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + result.put("list", getRAISpeciesListByParam(param)); + result.put("amount", getRAIAmountByParam(param)); + return result; + } + + @Override + public JSONObject getValidRNReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + JSONObject result = new JSONObject(); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + result.put("list", getValidRNListByParam(param)); + result.put("amount", getValidRNAmountByParam(param)); + return result; + } + + @Override + public JSONObject getValidPercentReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + JSONObject result = new JSONObject(); + result.put("list", getValidPercentListByPram(param)); + result.put("amount", getValidPercentAMountBuParam(param)); + return result; + } + + @Override + public JSONObject getValidRAIRNPtReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + + List list = new ArrayList<>(); + getArrayData(getRAISpeciesListByParam(param), Objects.requireNonNull(getRAIAmountByParam(param)), list, "RAI"); +// if (object != null) { +// String name = object.getString("name"); +// object.put("speciesName", name); +// object.put("name", "RAI"); +// list.add(object); +// } +// + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + getArrayData(Objects.requireNonNull(getValidRNListByParam(param)), Objects.requireNonNull(getValidRNAmountByParam(param)), list, "RN"); +// if (object != null) { +// String name = object.getString("name"); +// object.put("speciesName", name); +// object.put("name", "RN"); +// list.add(object); +// } +// + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + getArrayData(Objects.requireNonNull(getValidPercentListByPram(param)), Objects.requireNonNull(getValidPercentAMountBuParam(param)), list, "P"); +// if (object != null) { +// object.put("name", "P"); +// String name = object.getString("name"); +// object.put("speciesName", name); +// list.add(object); +// } + JSONObject result = new JSONObject(); + result.put("list", list); + return result; + } + + @Override + public JSONObject getRhythmReportByParam(JSONObject param) { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + param.put("speciesInIds", array); + } + param.remove("speciesIds"); + boolean flag = getDeviceIdsByBoundBox(param); + if (!flag) { + return new JSONObject(); + } + JSONObject result = new JSONObject(); + int dateType = param.getInteger("dateType"); + if (dateType == 1) { + result.put("list", getDayRhythmListByParam(param, 1, null, null)); + } else if (dateType == 2) { + result.put("list", getDayRhythmListByParam(param, 2, CalendarInterval.Day, "dd")); + } else if (dateType == 3) { + result.put("list", getDayRhythmListByParam(param, 3, CalendarInterval.Quarter, "QQQ")); + } else { + result.put("list", getDayRhythmListByParam(param, 4, CalendarInterval.Month, "MM")); + } + return result; + } + + @Override + public JSONObject getBirdPeriodStatByParam(JSONObject object) { + JSONObject result = new JSONObject(); + try { + int type = object.getInteger("type"); + CalendarInterval calendarInterval; + String format = ""; + if (type == 1) { + calendarInterval = CalendarInterval.Day; + format = "yyyy-MM-dd"; + } else if (type == 2) { + calendarInterval = CalendarInterval.Month; + format = "yyyy-MM"; + } else { + calendarInterval = CalendarInterval.Year; + format = "yyyy"; + } + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(object); + String finalFormat = format; + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", a -> a.dateHistogram(t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(calendarInterval) // + .format(finalFormat)// + .minDocCount(0)) + .aggregations("by_sum", sumAgg -> sumAgg + .sum(sm -> sm.field("count")) + ) + ).build(); +// SearchRequest searchRequest = new SearchRequest.Builder() +// .index(validIndexName) +// .query(q -> q.bool(boolQueryBuilder.build())) +// .size(0) +// .aggregations("by_record_time", agg -> agg +// .dateHistogram(t -> t +// .field("record_time") +// .timeZone("+08:00") +// .calendarInterval(CalendarInterval.Day) // 使用 calendarInterval 而不是 fixedInterval +// .format("yyyy-MM-dd") +// ) +// .aggregations("by_sum", sumAgg -> sumAgg +// .sum(sm -> sm.field("count")) +// ) +// ) +// .build(); + + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByRecordTime = aggregations.get("by_record_time"); + + List list = new ArrayList<>(); + if (aggByRecordTime != null) { + aggByRecordTime.dateHistogram().buckets().array().forEach(bucket -> { + JSONObject data = new JSONObject(); + data.put("time", bucket.keyAsString()); + Aggregate sumAgg = bucket.aggregations().get("by_sum"); + if (sumAgg != null) { + data.put("count", sumAgg.sum().value()); + } else { + data.put("count", 0); + } + list.add(data); + }); + result.put("list", list); + } else { + result.put("list", new ArrayList<>()); + } + } catch (Exception ex) { + logger.error("Elasticsearch查询失败: ", ex); + result.put("list", new ArrayList<>()); + } + return result; + } + + @Override + public JSONObject getSpeciesHabitatStatReportByParam(JSONObject param) { + JSONObject result = new JSONObject(); + result.put("list", createHabitatStatListByParam(param)); + return result; + } + + private List createHabitatStatListByParam(JSONObject queryParam) { + boolean flag = getDeviceIdsByBoundBox(queryParam); + if (!flag) { + return new ArrayList<>(); + } + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName).query(q -> q.bool(boolQueryBuilder.build())).size(0).aggregations("by_camera_id", + a -> a.terms(t -> t.field("device_id").size(10000)).aggregations("items_path", c -> createNestedAggregations(c, queryParam))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByCameraId = aggregations.get("by_camera_id"); + if (aggByCameraId != null) { +// log.info("====aggByCameraId=={}", aggByCameraId); + Map birdMap = new HashMap<>(); + Map deviceMap = new HashMap<>(); + List list = new ArrayList<>(); + aggByCameraId.lterms().buckets().array().forEach(cameraBucket -> { + Long cameraId = cameraBucket.key(); + JSONObject object = new JSONObject(); + object.put("deviceInfo", getDeviceInfoById(cameraId, deviceMap)); +// Map birdMap = new HashMap<>(); +// String key = "data_" + cameraId; + NestedAggregate nestedAggregation = cameraBucket.aggregations().get("items_path").nested(); + LongTermsAggregate aggBySpeciesId = nestedAggregation.aggregations().get("filtered_species").filter().aggregations().get("by_species_id").lterms(); + + if (aggBySpeciesId != null) { + JSONArray array = new JSONArray(); + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + JSONObject obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put("count", (int) bucket.aggregations().get("sum_value").sum().value()); + array.add(obj); + }); + object.put("list", array); + } + list.add(object); + }); + return list; + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + +// private List getDayRhythmListByParam(JSONObject param){ +// +// } + + private Map> initTimeMap(int type) { + Map> map = new HashMap<>(); + if (type == 1) { + for (int i = 0; i < 24; i++) { + String time = String.format("%02d", i); + map.put(time, null); + } + } else if (type == 2) { + for (int i = 1; i <= 31; i++) { + String time = String.format("%02d", i); + map.put(time, null); + } + } else if (type == 3) { + for (int i = 1; i <= 4; i++) { + String time = String.format("Q%d", i); + map.put(time, null); + } + } else if (type == 4) { + for (int i = 1; i <= 12; i++) { + String time = String.format("%02d", i); + map.put(time, null); + } + } + return map; + } + + private List getDayRhythmListByParam(JSONObject param, int type, CalendarInterval calendarInterval, String format) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = null; + if (type == 1) { + searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", agg -> agg.dateHistogram(t -> t.field("record_time").timeZone("+08:00")// + .calendarInterval(CalendarInterval.Hour).format("HH").offset(Time.of(tm -> tm.time("-6h"))))// + .aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)).aggregations("by_species_id", m -> m.terms(t -> t.field("species_id").size(10000))// + .aggregations("sum_value", sumAgg -> sumAgg.sum(sm -> sm.field("count")))))).build(); + } else { + searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", agg -> agg.dateHistogram(t -> t.field("record_time").timeZone("+08:00")// + .calendarInterval(calendarInterval).format(format))// + .aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)).aggregations("by_species_id", m -> m.terms(t -> t.field("species_id").size(10000))// + .aggregations("sum_value", sumAgg -> sumAgg.sum(sm -> sm.field("count")))))).build(); + } + + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByRecordTime = aggregations.get("by_record_time"); + List list = new ArrayList<>(); + Map birdMap = new HashMap<>(); + Map> map = initTimeMap(type);// new HashMap<>(); + if (aggByRecordTime != null) { + aggByRecordTime.dateHistogram().buckets().array().forEach(bucket -> { +// JSONObject obj = new JSONObject(); +// long time = bucket.key(); + String time = bucket.keyAsString(); +// log.info("time: {}", time); +// obj.put("time", time); + Map subMap = map.get(time); + if (subMap == null) { + subMap = new HashMap<>(); + } + Aggregate aggByMedia = bucket.aggregations().get("by_media_type"); + if (aggByMedia != null) { + Map finalSubMap = subMap; + aggByMedia.lterms().buckets().array().forEach(mediaBucket -> { + String code = "data" + mediaBucket.key(); + Aggregate aggBySpecies = mediaBucket.aggregations().get("by_species_id"); + if (aggBySpecies != null) { +// JSONArray array = new JSONArray(); + aggBySpecies.lterms().buckets().array().forEach(subBucket -> { + long speciesId = subBucket.key(); + JSONObject subObj = finalSubMap.get(speciesId); + if (subObj != null) { + int count = (int) subBucket.aggregations().get("sum_value").sum().value(); + Integer count1 = subObj.getInteger(code); + if (count1 != null) { + subObj.put(code, count1 + count); + } else { + subObj.put(code, count); + } + } else { + subObj = new JSONObject(); + subObj.put("speciesId", speciesId); + subObj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + subObj.put(code, (int) subBucket.aggregations().get("sum_value").sum().value()); + } + finalSubMap.put(speciesId, subObj); + }); + map.put(time, finalSubMap); + } + }); + } +// list.add(obj); + }); + } + + map.keySet().forEach(key -> { + JSONObject obj = new JSONObject(); + if (map.get(key) != null) { + obj.put("list", new ArrayList<>(map.get(key).values())); + } else { + obj.put("list", new ArrayList<>()); + } + key = key.replaceAll("Q", ""); + obj.put("time", Integer.parseInt(key)); + list.add(obj); + }); + list.sort(new Comparator() { + @Override + public int compare(JSONObject o1, JSONObject o2) { + return o1.getInteger("time") - o2.getInteger("time"); + } + }); + return list; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private void getArrayData(List array, JSONObject obj, List list, String type) { + Integer amount1 = obj.getInteger("data2"); + Integer amount2 = obj.getInteger("data3"); + DecimalFormat df = new DecimalFormat("#.00"); + if (!array.isEmpty()) { + JSONObject object = new JSONObject(); + object.put("speciesName", array.get(0).get("name")); + if (amount1 != null) { + Integer count = array.get(0).getInteger("data2"); + if (count != null) { + object.put("data2", Double.parseDouble(df.format(count * 100.0 / amount1))); + } + } + + if (amount2 != null) { + Integer count = array.get(0).getInteger("data3"); + if (count != null) { + object.put("data3", Double.parseDouble(df.format(count * 100.0 / amount2))); + } + } + object.put("name", type); + list.add(object); +// return object; + } +// return null; + } + + private List getValidPercentListByPram(JSONObject param) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_species_id", agg -> agg.terms(t -> t.field("species_id").size(10000)).aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)).aggregations("by_device_id", m -> m.terms(t -> t.field("device_id").size(10000))// + ))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + Map birdMap = new HashMap<>(); + if (aggBySpeciesId != null) { + List list = new LinkedList<>(); + aggBySpeciesId.lterms().buckets().array().forEach(bucket -> { + long speciesId = bucket.key(); + Aggregate aggByMediaType = bucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + JSONObject obj = new JSONObject(); + obj.put("id", speciesId); + obj.put("name", getSpeciesNameByName(speciesId, birdMap)); + aggByMediaType.lterms().buckets().array().forEach(mediaBucket -> { + String key = "data" + mediaBucket.key(); + Aggregate aggByDeviceId = mediaBucket.aggregations().get("by_device_id"); + obj.put(key, aggByDeviceId.lterms().buckets().array().size()); + }); + list.add(obj); + } + }); + return list; + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONObject getValidPercentAMountBuParam(JSONObject param) { + param.remove("speciesInIds"); + JSONObject result = new JSONObject(); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_media_type", agg -> agg.terms(t -> t.field("media_type").size(10)).aggregations("by_device_id", m -> m.terms(t -> t.field("device_id").size(100000))// + )).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByMediaType = aggregations.get("by_media_type"); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(bucket -> { + String key = "data" + bucket.key(); + Aggregate aggById = bucket.aggregations().get("by_device_id"); + if (aggById != null) { + result.put(key, aggById.lterms().buckets().array().size()); + } + }); + } + } catch (Exception ex) { + logger.error("error", ex); + } + return result; + } + + private List getValidRNListByParam(JSONObject param) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_species_id", agg -> agg.terms(t -> t.field("species_id").size(10000)).aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)) +// .aggregations("by_media_id", m -> m.terms(t -> t.field("id"))// + )).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + Map birdMap = new HashMap<>(); + if (aggBySpeciesId != null) { + List list = new LinkedList<>(); + aggBySpeciesId.lterms().buckets().array().forEach(bucket -> { + long speciesId = bucket.key(); + Aggregate aggByMediaType = bucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + JSONObject obj = new JSONObject(); + obj.put("id", speciesId); + obj.put("name", getSpeciesNameByName(speciesId, birdMap)); + aggByMediaType.lterms().buckets().array().forEach(mediaBucket -> { + String key = "data" + mediaBucket.key(); + obj.put(key, mediaBucket.docCount()); +// Aggregate aggBYMediaId = mediaBucket.aggregations().get("by_media_id"); +// obj.put(key, aggBYMediaId.lterms().buckets().array().size()); + }); + list.add(obj); + } + }); + return list; + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONObject getValidRNAmountByParam(JSONObject param) { + param.remove("speciesInIds"); + JSONObject result = new JSONObject(); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_media_type", agg -> agg.terms(t -> t.field("media_type").size(10)) +// .aggregations("by_media_id", m -> m.terms(t -> t.field("id").size(100000))// + ).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByMediaType = aggregations.get("by_media_type"); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(bucket -> { + String key = "data" + bucket.key(); + result.put(key, bucket.docCount()); +// Aggregate aggById = bucket.aggregations().get("by_media_id"); +// if (aggById != null) { +// result.put(key, aggById.lterms().buckets().array().size()); +// } else { +// result.put(key, 0); +// } + }); + } + } catch (Exception ex) { + logger.error("error", ex); + } + return result; + } + + private List getRAISpeciesListByParam(JSONObject param) { +// log.info("=============rai param={}", param); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + List list = new ArrayList<>(); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_species_id", agg -> agg.terms(t -> t.field("species_id").size(10000)).aggregations("by_media_type", m -> m.terms(t -> t.field("media_type").size(10)).aggregations("by_record_time", d -> d.dateHistogram(t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(CalendarInterval.Day) // + .format("yyyy-MM-dd")// + .minDocCount(0))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(speciesTermsBucket -> { + JSONObject obj = new JSONObject(); + long id = speciesTermsBucket.key(); + obj.put("id", id); + obj.put("name", birdSpeciesService.getSpeciesNameById(id)); + Aggregate aggByMediaType = speciesTermsBucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(mediaTermsBucket -> { + String key = "data" + mediaTermsBucket.key(); + int count = mediaTermsBucket.aggregations().get("by_record_time").dateHistogram().buckets().array().size(); + obj.put(key, count); + }); + } + list.add(obj); + }); + } + } catch (Exception ex) { + logger.error("error", ex); + } + return list; + } + + private JSONObject getRAIAmountByParam(JSONObject param) { + param.remove("speciesInIds"); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", agg -> agg.terms(t -> t.field("device_id").size(10000)).aggregations("by_media_type", m -> m.terms(t -> t.field("media_type").size(10)).aggregations("by_record_time", d -> d.dateHistogram(t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(CalendarInterval.Day) // + .format("yyyy-MM-dd")// + .minDocCount(0))))).build(); + try { + JSONObject object = new JSONObject(); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByDeviceId = aggregations.get("by_device_id"); + if (aggByDeviceId != null) { + aggByDeviceId.lterms().buckets().array().forEach(longTermsBucket -> { + Aggregate aggByMediaType = longTermsBucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(mediaTermsBucket -> { + String key = "data" + mediaTermsBucket.key(); + Integer count = object.getInteger(key); + if (count == null) { + count = 0; + } + Aggregate aggByRecordTime = mediaTermsBucket.aggregations().get("by_record_time"); + count += aggByRecordTime.dateHistogram().buckets().array().size(); + object.put(key, count); + }); + } + }); + } + return object; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private Aggregation.Builder.ContainerBuilder createNestedAggregations(Aggregation.Builder c, JSONObject param) { + return c.nested(n -> n.path("items")).aggregations("filtered_species", f -> { + JSONArray array = param.getJSONArray("speciesIds"); + if (array != null && !array.isEmpty()) { + List idList = new ArrayList<>(); + for (Object id : array) { + idList.add(Long.parseLong(id.toString())); + } + List fieldValues = idList.stream().map(FieldValue::of).collect(Collectors.toList()); + return f.filter(ft -> ft.terms(t2 -> t2.field("items.species_id").terms(v -> v.value(fieldValues)))).aggregations("by_species_id", + d -> d.terms(t2 -> t2.field("items.species_id").size(10000)).aggregations("sum_value", + sumAgg -> sumAgg.sum(m -> m.field("items.count")))); + } else { + String excludeIds = param.getString("excludeIds"); + if (excludeIds != null && !excludeIds.isEmpty()) { + String[] ids = excludeIds.split("_"); + List idList = new ArrayList<>(); + for (String id : ids) { + idList.add(Long.parseLong(id)); + } + List fieldValues = idList.stream().map(FieldValue::of).collect(Collectors.toList()); + + return f.filter(ft -> ft.bool(b -> b.mustNot(mn -> mn.terms(t2 -> t2.field("items.species_id") + .terms(v -> v.value(fieldValues)))))).aggregations("by_species_id", + d -> d.terms(t2 -> t2.field("items.species_id").size(10000)) + .aggregations("sum_value", sumAgg -> sumAgg.sum(m -> m.field("items.count")))); + + } else { + return f.aggregations("by_species_id", d -> d.terms(t2 -> t2.field("items.species_id").size(10000)) + .aggregations("sum_value", sumAgg -> sumAgg.sum(m -> m.field("items.count")))); + } + + } + }); + } + + private List dataValidStatByTime(JSONObject param, CalendarInterval calendarInterval, String dateFormat) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", a -> a.dateHistogram(t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(calendarInterval) // + .format(dateFormat)// + .minDocCount(0)).aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)) + .aggregations("by_species_id", d -> d.terms(s -> s.field("species_id").size(10000)) + .aggregations("sum_value", sumAgg -> sumAgg.sum(m -> m.field("count")))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + List array = new ArrayList<>(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByRecordTime = aggregations.get("by_record_time"); +// log.info("=====aggByRecordTime==========={}", aggByRecordTime); + if (aggByRecordTime != null) { + aggByRecordTime.dateHistogram().buckets().array().forEach(dateBucket -> { + String time = dateBucket.keyAsString(); + JSONObject jobj = new JSONObject(); + jobj.put("time", time); + Aggregate aggByMediaType = dateBucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + Map map = new HashMap<>(); + aggByMediaType.lterms().buckets().array().forEach(typeBucket -> { + int mediaType = (int) typeBucket.key(); + Map birdMap = new HashMap<>(); + String key = "data" + mediaType; + LongTermsAggregate aggBySpeciesId = typeBucket.aggregations().get("by_species_id").lterms(); + if (aggBySpeciesId != null) { + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + JSONObject obj = map.get(speciesId); + if (obj == null) { + obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } else { + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } + map.put(speciesId, obj); + }); + } + }); + jobj.put("items", new ArrayList<>(map.values())); + } + array.add(jobj); + }); + return array; + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private List dataStatByTime(JSONObject param, CalendarInterval calendarInterval, String dateFormat) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", a -> a.dateHistogram(t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(calendarInterval) // + .format(dateFormat)// + .minDocCount(0)).aggregations("by_media_type", mt -> mt.terms(t -> t.field("media_type").size(10)) + + .aggregations("items_path", c -> createNestedAggregations(c, param)))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + List array = new ArrayList<>(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByRecordTime = aggregations.get("by_record_time"); +// log.info("=====aggByRecordTime==========={}", aggByRecordTime); + if (aggByRecordTime != null) { + aggByRecordTime.dateHistogram().buckets().array().forEach(dateBucket -> { + String time = dateBucket.keyAsString(); + JSONObject jobj = new JSONObject(); + jobj.put("time", time); + Aggregate aggByMediaType = dateBucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + + Map map = new HashMap<>(); + aggByMediaType.lterms().buckets().array().forEach(typeBucket -> { +// log.info("====typeBucket===={}", typeBucket); + int mediaType = (int) typeBucket.key(); + Map birdMap = new HashMap<>(); + String key = "data" + mediaType; +// log.info("=====items_path========{}", typeBucket.aggregations().get("items_path")); + NestedAggregate nestedAggregation = typeBucket.aggregations().get("items_path").nested(); + LongTermsAggregate aggBySpeciesId = nestedAggregation.aggregations().get("filtered_species").filter().aggregations().get("by_species_id").lterms(); + if (aggBySpeciesId != null) { + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + JSONObject obj = map.get(speciesId); + if (obj == null) { + obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } else { + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } + map.put(speciesId, obj); + }); + } + }); + jobj.put("items", new ArrayList<>(map.values())); + } + array.add(jobj); + }); + } + return array; + + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONArray getSpeciesDailyStatByParam(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_record_time", Aggregation.of(a -> a.dateHistogram((t -> t.field("record_time")// + .timeZone("+08:00")// + .calendarInterval(CalendarInterval.Day) // + .format("yyyy-MM-dd")// + )).aggregations("sum_count_field", Aggregation.of(c -> c.sum(SumAggregation.of(t -> t.field("count"))))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByRecordTime = aggregations.get("by_record_time"); + if (aggByRecordTime != null) { + aggByRecordTime.dateHistogram().buckets().array().forEach(bucket -> { + String time = bucket.keyAsString(); +// log.info("==============={}", bucket); + JSONObject object = new JSONObject(); + object.put("date", time); + Aggregate sumAgg = bucket.aggregations().get("sum_count_field"); + object.put("count", (int) sumAgg.sum().value()); + array.add(object); + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONArray getSpeciesStatByParam(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_species_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("species_id").size(10000) // 定义返回的分组数量,例如10个 + )).aggregations("sum_count_field", Aggregation.of(c -> c.sum(SumAggregation.of(t -> t.field("count"))))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(bucket -> { + JSONObject object = new JSONObject(); + long speciesId = bucket.key(); + if (speciesId > 0L) { + object.put("speciesId", speciesId); + Aggregate sumAgg = bucket.aggregations().get("sum_count_field"); + object.put("count", (int) sumAgg.sum().value()); + object.put("speciesName", birdSpeciesService.getSpeciesNameById(speciesId)); + array.add(object); + } + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private List getDeviceListByParam(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("device_id").size(10000))) // + )).build(); + + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + List deviceList = new ArrayList<>(); + Map aggregations = searchResponse.aggregations(); + Aggregate byDeviceId = aggregations.get("by_device_id"); + if (byDeviceId != null) { + byDeviceId.lterms().buckets().array().forEach(deviceBucket -> { + long deviceId = deviceBucket.key(); + deviceList.add(deviceId); + }); + } + return deviceList; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONArray getSpeciesAmountInCamera(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("device_id").size(10000))) // + .aggregations("by_media_type", Aggregation.of(mt -> mt.terms(TermsAggregation.of(t -> t.field("media_type").size(10))) // + .aggregations("items_path", Aggregation.of(n -> n.nested(ne -> ne.path("items")) // + .aggregations("by_species_id", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("items.species_id").size(10000))))))))))).build(); + + try { + Map birdMap = new HashMap<>(); + Map deviceMap = new HashMap<>(); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate byDeviceId = aggregations.get("by_device_id"); + if (byDeviceId != null) { + byDeviceId.lterms().buckets().array().forEach(deviceBucket -> { + long deviceId = deviceBucket.key(); + JSONObject object = new JSONObject(); + object.put("deviceId", deviceId); + object.put("device", getDeviceInfoById(deviceId, deviceMap)); + + Aggregate aggByMediaType = deviceBucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + List imgList = new ArrayList<>(); + List vodList = new ArrayList<>(); + + aggByMediaType.lterms().buckets().array().forEach(mediaBucket -> { + int mediaType = (int) mediaBucket.key(); + Aggregate itemsPathAgg = mediaBucket.aggregations().get("items_path"); + if (itemsPathAgg != null) { + Aggregate aggBySpeciesId = itemsPathAgg.nested().aggregations().get("by_species_id"); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(speciesBucket -> { + JSONObject sub = new JSONObject(); + long speciesId = speciesBucket.key(); + sub.put("speciesId", speciesId); + sub.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + sub.put("count", speciesBucket.docCount()); + // 根据媒体类型分类存放,2通常代表图片,3通常代表视频 + if (mediaType == 2) { + imgList.add(sub); + } else { + vodList.add(sub); + } + }); + } + } + }); + + object.put("imgList", imgList); + object.put("vodList", vodList); + } + array.add(object); + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + + private String getSpeciesNameByName(long id, Map map) { + String name = map.get(id); + if (name != null) { + return name; + } else { + name = birdSpeciesService.getSpeciesNameById(id); + map.put(id, name); + return name; + } + } + + + private long getCameraAmountByParam(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.cardinality(CardinalityAggregation.of(t -> t.field("device_id"))))).build(); + + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + return aggregations.get("by_device_id").cardinality().value(); +// Aggregate aggByDevice = aggregations.get("by_species_id"); +// if (aggByDevice != null) { +// return aggByDevice.lterms().buckets().array().size(); +// } else { +// return 0; +// } + } catch (Exception ex) { + logger.error("==error===", ex); + } + return 0; + } + + + private JSONObject calcDateRange(JSONObject param) { + JSONObject queryParam = new JSONObject(); + String str = param.getString("calcMonth"); + boolean flag = false; + String dateFrom = ""; + String dateTo = ""; + if (str != null && !str.isEmpty()) { + str = str + "-01"; + LocalDate localDate = LocalDate.parse(str); + dateFrom = localDate.with(TemporalAdjusters.firstDayOfMonth()).toString(); + dateTo = localDate.with(TemporalAdjusters.lastDayOfMonth()).toString(); + flag = true; + } else { + str = param.getString("calcQuarter"); + if (str != null && !str.isEmpty()) { + LocalDate localDate = LocalDate.parse(str); + dateFrom = localDate.with(TemporalAdjusters.firstDayOfMonth()).toString(); + localDate = localDate.plusMonths(2); + dateTo = localDate.with(TemporalAdjusters.lastDayOfMonth()).toString(); + flag = true; + } else { + str = param.getString("calcYear"); + if (str != null && !str.isEmpty()) { + dateFrom = str + "-01-01"; + dateTo = str + "-12-31"; + flag = true; + } + } + } + if (!flag) { + LocalDate localDate = LocalDate.now(); + dateFrom = localDate.with(TemporalAdjusters.firstDayOfMonth()).toString(); + dateTo = localDate.with(TemporalAdjusters.lastDayOfMonth()).toString(); + } + queryParam.put("dateFrom", dateFrom); + queryParam.put("dateTo", dateTo); + queryParam.put("speciesId", param.getLong("speciesId")); + return queryParam; + } + + + private JSONArray getResultSpeciesStat(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_species_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("species_id").size(10000) // 定义返回的分组数量,例如10个 + )).aggregations("by_media_type", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("media_type").size(10))))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggBySpeciesId = aggregations.get("by_species_id"); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(bucket -> { + JSONObject object = new JSONObject(); + long speciesId = bucket.key(); + if (speciesId > 0L) { + object.put("speciesId", speciesId); + Aggregate aggByMediaType = bucket.aggregations().get("by_media_type"); + JSONArray subArray = new JSONArray(); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(subBucket -> { + JSONObject subObject = new JSONObject(); + subObject.put("mediaType", subBucket.key()); + subObject.put("count", subBucket.docCount()); + subArray.add(subObject); + }); + object.put("speciesName", birdSpeciesService.getSpeciesNameById(speciesId)); + object.put("list", subArray); + array.add(object); + } + } + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private List getResultValidAllStatListByParam(JSONObject queryParam) { + queryParam.remove("speciesIds"); + boolean flag = getDeviceIdsByBoundBox(queryParam); + if (!flag) { + return new ArrayList<>(); + } + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_media_type", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("media_type").size(10) // + ))// + .aggregations("by_species_id", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("species_id").size(10000))).aggregations("sum_value",// + sumAgg -> sumAgg.sum(m -> m.field("count"))))))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByMediaType = aggregations.get("by_media_type"); + if (aggByMediaType != null) { + Map map = new HashMap<>(); + aggByMediaType.lterms().buckets().array().forEach(typeBucket -> { + int mediaType = (int) typeBucket.key(); + Map birdMap = new HashMap<>(); + String key = "data" + mediaType; + LongTermsAggregate aggBySpeciesId = typeBucket.aggregations().get("by_species_id").lterms(); + if (aggBySpeciesId != null) { + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + JSONObject obj = map.get(speciesId); + if (obj == null) { + obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } else { + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } + map.put(speciesId, obj); + }); + } + }); + return new ArrayList<>(map.values()); + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private JSONArray getResultValidStatListByParam(JSONObject queryParam) { +// log.info("===========query==={}", queryParam); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(validIndexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("device_id").size(100) // 定义返回的分组数量 + ))// + .aggregations("by_media_type", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("media_type").size(10))).aggregations("by_species_id", Aggregation.of(s -> s.terms(TermsAggregation.of(t -> t.field("species_id")// + .size(1000)// + ))// + .aggregations("by_record_time",// + Aggregation.of(d -> d.dateHistogram(histo -> histo.field("record_time")// + .timeZone("+08:00").calendarInterval(CalendarInterval.Day)// + .format("yyyy-MM-dd")// + )// + .aggregations("sum_value",// + sumAgg -> sumAgg.sum(m -> m.field("count")))// + ))// + )// + ))))).build(); + try { + Map deviceMap = new HashMap<>(); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByDeviceId = aggregations.get("by_device_id"); + Map birdMap = new HashMap<>(); + if (aggByDeviceId != null) { + Set daySet = new HashSet<>(); + aggByDeviceId.lterms().buckets().array().forEach(bucket -> { + Long deviceId = bucket.key(); + daySet.clear(); + Aggregate aggByMediaType = bucket.aggregations().get("by_media_type"); + List imgList = new ArrayList<>(); + List vodList = new ArrayList<>(); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(bucket01 -> { + int mediaType = (int) bucket01.key(); + Aggregate aggBySpeciesId = bucket01.aggregations().get("by_species_id"); + if (aggBySpeciesId != null) { +// Map map = new HashMap<>(); + JSONObject subObj = new JSONObject(); + aggBySpeciesId.lterms().buckets().array().forEach(bucket02 -> { + long speciesId = bucket02.key(); + subObj.put("speciesId", speciesId); + subObj.put("name", getSpeciesNameByName(speciesId, birdMap)); + Aggregate aggByTime = bucket02.aggregations().get("by_record_time"); + if (aggByTime != null) { + +// subObj.put("days", aggByTime.dateHistogram().buckets().array().size()); + +// log.info("======size========{}", aggByTime.dateHistogram().buckets().array().size()); + aggByTime.dateHistogram().buckets().array().forEach(bucket03 -> { + String time = bucket03.keyAsString(); + if (time != null && time.length() >= 10) { + daySet.add(time.substring(0, 10)); + } else { + daySet.add(time); + } +// log.info("==time======{}", bucket03.keyAsString()); + subObj.put("count", (int) bucket03.aggregations().get("sum_value").sum().value()); + subObj.put("mediaCount", bucket03.docCount()); + }); + } + + + }); + if (mediaType == 2) { + imgList.add(subObj); + } else { + vodList.add(subObj); + } + } + }); + } + JSONObject object = new JSONObject(); + object.put("deviceId", deviceId); + object.put("imgList", imgList); + object.put("vodList", vodList); + object.put("days", daySet.size()); + object.put("device", getDeviceInfoById(deviceId, deviceMap)); + array.add(object); + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private DeviceInfo getDeviceInfoById(Long id, Map map) { + DeviceInfo data = map.get(id); + if (data != null) { + return data; + } else { + data = deviceInfoService.getDeviceInfoForElasticById(id); + map.put(id, data); + return data; + } + } + +// private boolean getDeviceIdsByDeptId(JSONObject queryParam) { +// Long deptId = queryParam.getLong("deptId"); +// if (deptId == null) { +// return true; +// } +// List deptIds = sysDeptMapper.getDeptSubIdsById(deptId); +// List deviceIds = deviceInfoService.getDeviceIdsInList(deptIds); +// if (deviceIds == null || deviceIds.isEmpty()) { +// return false; +// } else { +// queryParam.put("deviceIds", deviceIds); +// return true; +// } +// } + + private List getCameraStatListByParam(JSONObject queryParam) { + boolean flag = getDeviceIdsByBoundBox(queryParam); + if (!flag) { + return new ArrayList<>(); + } + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); +// SearchRequest searchRequest = new SearchRequest.Builder() +// .index(indexName) // +// .query(q -> q.bool(boolQueryBuilder.build())) +// .size(0) // +// .aggregations("by_media_type", a -> +// a.terms(t -> t +// .field("media_type") +// )// +// .aggregations("items_path", +// c -> c.nested( +// n -> n.path("items") +// )// +// .aggregations("by_species_id",// +// d -> d.terms(t -> +// t.field("items.species_id").size(10000) +// )// +// .aggregations("sum_value",// +// sumAgg -> sumAgg.sum(m -> m.field("items.count")) +// ) +// ) +// ) +// ).build(); + + + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName).query(q -> q.bool(boolQueryBuilder.build())).size(0).aggregations("by_media_type", + a -> a.terms(t -> t.field("media_type")).aggregations("items_path", c -> createNestedAggregations(c, queryParam))).build(); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByMediaType = aggregations.get("by_media_type"); + + if (aggByMediaType != null) { +// log.info("====aggByMediaType=={}", aggByMediaType); + Map map = new HashMap<>(); + aggByMediaType.lterms().buckets().array().forEach(typeBucket -> { + int mediaType = (int) typeBucket.key(); + Map birdMap = new HashMap<>(); + String key = "data" + mediaType; + NestedAggregate nestedAggregation = typeBucket.aggregations().get("items_path").nested(); + LongTermsAggregate aggBySpeciesId = nestedAggregation.aggregations().get("filtered_species").filter().aggregations().get("by_species_id").lterms(); + + if (aggBySpeciesId != null) { + aggBySpeciesId.buckets().array().forEach(bucket -> { + Long speciesId = bucket.key(); + + JSONObject obj = map.get(speciesId); + if (obj == null) { + obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("speciesName", getSpeciesNameByName(speciesId, birdMap)); + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } else { + obj.put(key, (int) bucket.aggregations().get("sum_value").sum().value()); + } + map.put(speciesId, obj); + }); + } + }); + return new ArrayList<>(map.values()); + } + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + + private JSONArray getResultStatListByParam(JSONObject queryParam) { +// log.info("==========queryParam========{}", queryParam); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_device_id", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("device_id").size(10000) // 定义返回的分组数量 + ))// + .aggregations("by_media_type", Aggregation.of(c -> c.terms(TermsAggregation.of(t -> t.field("media_type").size(10)))// + .aggregations("by_media_id",// + Aggregation.of(d -> d.terms(TermsAggregation.of(t -> t.field("media_id").size(10000)))// + .aggregations("by_species_id",// + Aggregation.of(m -> m.terms(TermsAggregation.of(t -> t.field("species_id").size(10000)))))// + )))))).build(); + try { + Map deviceMap = new HashMap<>(); + Map birdMap = new HashMap<>(); + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate aggByDeviceId = aggregations.get("by_device_id"); +// log.info("=====aggByDeviceId========={}", aggByDeviceId); + if (aggByDeviceId != null) { + aggByDeviceId.lterms().buckets().array().forEach(bucket -> { + JSONObject object = new JSONObject(); + long deviceId = bucket.key(); + Aggregate aggByMediaType = bucket.aggregations().get("by_media_type"); + if (aggByMediaType != null) { + aggByMediaType.lterms().buckets().array().forEach(typeBucket -> { + int mediaType = (int) typeBucket.key(); + Aggregate aggByMediaId = typeBucket.aggregations().get("by_media_id"); + if (aggByMediaId != null) { + Map map = new HashMap<>(); + int count = aggByMediaId.lterms().buckets().array().size(); + aggByMediaId.lterms().buckets().array().forEach(mediaBucket -> { + Aggregate aggBySpeciesId = mediaBucket.aggregations().get("by_species_id"); + if (aggBySpeciesId != null) { + aggBySpeciesId.lterms().buckets().array().forEach(bucket01 -> { + long speciesId = bucket01.key(); + map.merge(speciesId, 1, Integer::sum); + }); + } + }); + if (mediaType == 2) { + object.put("imageList", convertSpeciesMap(map, birdMap)); + object.put("imageCount", count); + + } else { + object.put("videoList", convertSpeciesMap(map, birdMap)); + object.put("videoCount", count); + } + } + }); + object.put("deviceId", deviceId); + object.put("device", getDeviceInfoById(deviceId, deviceMap)); + array.add(object); + } + }); + } + return array; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + private List convertSpeciesMap(Map map, Map birdMap) { + List list = new ArrayList<>(); + map.keySet().forEach(speciesId -> { + if (speciesId != 0L) { + String name = getSpeciesNameByName(speciesId, birdMap); + JSONObject obj = new JSONObject(); + obj.put("speciesId", speciesId); + obj.put("name", name); + obj.put("count", map.get(speciesId)); + list.add(obj); + } + }); + return list; + } + + private JSONArray getResultStatAmount(JSONObject queryParam) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(queryParam); + SearchRequest searchRequest = new SearchRequest.Builder().index(indexName) // + .query(q -> q.bool(boolQueryBuilder.build())).size(0) // + .aggregations("by_media_type", Aggregation.of(a -> a.terms(TermsAggregation.of(t -> t.field("media_type").size(100) // 定义返回的分组数量,例如10个 + )).aggregations("by_media_id", Aggregation.of(c -> c.cardinality(CardinalityAggregation.of(card -> card.field("media_id") // + )))))).build(); + + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + + JSONArray array = new JSONArray(); + Map aggregations = searchResponse.aggregations(); + Aggregate categoryGroup = aggregations.get("by_media_type"); + + if (categoryGroup != null) { + categoryGroup.lterms().buckets().array().forEach(bucket -> { + long category = bucket.key(); + long docCount = bucket.docCount(); + JSONObject obj = new JSONObject(); + // 获取不重复的 user_id 数量 + long mediaIdCount = bucket.aggregations().get("by_media_id").cardinality().value(); + obj.put("mediaType", category); + if (category == 2) { + obj.put("mediaTypeName", "照片"); + } else if (category == 3) { + obj.put("mediaTypeName", "视频"); + } + obj.put("count", mediaIdCount); + array.add(obj); +// System.out.println("Category: " + category + +// ", Document Count: " + docCount + +// ", Unique User Count: " + mediaIdCount); + }); + return array; + } + + } catch (IOException ex) { + logger.error("getResultRecordStatReportByParam", ex); + } + return null; + } + + + private JSONArray getSpeciesDailyAmountByDeviceId(JSONObject param) { + Map map = new HashMap<>(); + Long areaId = param.getLong("areaId"); + map.put("areaId", areaId); + List list = esResultRecordDetailsMapper.getDeviceListByParam(map); + JSONArray deviceDataArray = new JSONArray(); + list.forEach(item -> { + param.put("deviceId", item.getId()); + param.put("recordType", 1); + try { + JSONObject obj = new JSONObject(); + obj.put("name", item.getName()); + obj.put("id", item.getId()); + JSONArray array = CommonEsUtils.getSpeciesAggregateList(param, indexName, elasticsearchClient, CalendarInterval.Hour); + if (!array.isEmpty()) { + JSONArray ids = new JSONArray(); + for (Object obj1 : array) { + JSONObject jsonObject = (JSONObject) obj1; + ids.add(jsonObject.getLong("speciesId")); + } + param.put("ids", ids); + } else { + array = new JSONArray(); + } + param.put("recordType", 2); + JSONArray array1 = CommonEsUtils.getSpeciesAggregateList(param, indexName, elasticsearchClient, CalendarInterval.Hour); + array.addAll(array1); + obj.put("list", array); + deviceDataArray.add(obj); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + }); + return deviceDataArray; + } + + private Object[] compactJSONArray(JSONArray array) { + Map map = new HashMap<>(); +// JSONArray compactArray = new JSONArray(); + for (Object obj : array) { + JSONObject obj1 = (JSONObject) obj; + JSONArray a1 = obj1.getJSONArray("list"); + for (Object obj2 : a1) { + JSONObject obj3 = (JSONObject) obj2; + Long speciesId = obj3.getLong("speciesId"); + JSONObject data = map.get(speciesId); + int[] amountArr = null; + if (data == null) { + data = new JSONObject(); + amountArr = new int[11]; + data.put("speciesId", speciesId); + } else { + amountArr = (int[]) data.get("array"); + } + int hour = obj3.getInteger("hour"); + hour -= 8; + if (hour >= 0 && hour <= 11) { + amountArr[hour] += obj3.getInteger("count"); + } + data.put("array", amountArr); + map.put(speciesId, data); + } + } + return map.values().toArray(); + } + + + private JSONArray getSpeciesHabitatAggregateList(JSONObject param) throws IOException { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0).query(q -> q.bool(boolQueryBuilder.build())).aggregations("by_speciesId", a -> a.terms(TermsAggregation.of(ta -> ta.field("species_id").size(10000)))// + .aggregations("by_habitat", b -> b.terms(TermsAggregation.of(ta -> ta.field("habitat")))// + .aggregations("by_hour", c -> c.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time")// + .timeZone("+08:00").calendarInterval(CalendarInterval.Hour)))// + .aggregations("sum_count_field", s -> s.sum(SumAggregation.of(sa -> sa.field("count")))))))); + + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate bySpeciesAgg = aggs.get("by_speciesId"); + JSONArray jsonArray = new JSONArray(); + + bySpeciesAgg.lterms().buckets().array().forEach(speciesBucket -> { + Aggregate byHabitatAgg = speciesBucket.aggregations().get("by_habitat"); + byHabitatAgg.sterms().buckets().array().forEach(habitatBucket -> { + Aggregate byHourAgg = habitatBucket.aggregations().get("by_hour"); + JSONObject item = new JSONObject(); + item.put("habitat", habitatBucket.key().stringValue()); + item.put("speciesId", speciesBucket.key()); + final int[] max = {0}; + byHourAgg.dateHistogram().buckets().array().forEach(hourBucket -> { + Aggregate sumAgg = hourBucket.aggregations().get("sum_count_field"); + if (max[0] < (int) sumAgg.sum().value()) { + max[0] = (int) sumAgg.sum().value(); + } + }); + item.put("count", max[0]); + jsonArray.add(item); + }); + }); + return jsonArray; + } + + private JSONArray filterArrayBySpeciesId(Object[] array, Long id) { +// JSONArray filterArray = new JSONArray(); + for (Object obj : array) { + JSONObject jobj = (JSONObject) obj; + Long speciesId = jobj.getLong("speciesId"); + if (Objects.equals(speciesId, id)) { +// filterArray.add(); + return jobj.getJSONArray("array"); + } + } + return new JSONArray(); + } + +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsResultRecordDetailsServiceImpl.java b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsResultRecordDetailsServiceImpl.java new file mode 100644 index 0000000..d4d5ff0 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/service/impl/EsResultRecordDetailsServiceImpl.java @@ -0,0 +1,1217 @@ +package net.hzvtc.service.impl; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.Script; +import co.elastic.clients.elasticsearch._types.ScriptLanguage; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.*; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery; +import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.TrackHits; +import co.elastic.clients.json.JsonData; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.github.pagehelper.PageInfo; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.hzvtc.area.domain.ProtectedAreaSpecies; +import net.hzvtc.area.service.ProtectedAreaSpeciesService; +import net.hzvtc.bird.domain.BirdSpecies; +import net.hzvtc.bird.service.BirdMediaService; +import net.hzvtc.bird.service.BirdSpeciesService; +import net.hzvtc.common.constant.HttpStatus; +import net.hzvtc.common.core.page.TableDataInfo; +import net.hzvtc.common.utils.PageUtils; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.data.check.domain.FavoriteMedia; +import net.hzvtc.data.check.domain.ResultMedia; +import net.hzvtc.data.check.domain.ResultRecordDetails; +import net.hzvtc.data.check.mapper.ResultRecordDetailsMapper; +import net.hzvtc.data.check.service.FavoriteMediaService; +import net.hzvtc.device.domain.DeviceInfo; +import net.hzvtc.device.service.DeviceInfoService; +import net.hzvtc.echart.domain.vo.BirdStatDataVo; +import net.hzvtc.es.domain.EsResultRecordDetails; +import net.hzvtc.es.domain.EsResultRecordDetailsVo; +import net.hzvtc.es.mapper.EsResultRecordDetailsMapper; +import net.hzvtc.service.EsResultRecordDetailsService; +import net.hzvtc.system.mapper.SysDeptMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + + +@Service("esResultRecordDetailsService") +@RequiredArgsConstructor(onConstructor = @__(@Lazy)) +@Slf4j +public class EsResultRecordDetailsServiceImpl implements EsResultRecordDetailsService { + private static final Logger logger = LoggerFactory.getLogger(EsResultRecordDetailsService.class); + // @Resource(name="clientByPasswd") +// @Qualifier("clientByPasswd") + private final ElasticsearchClient elasticsearchClient; + + private final BirdSpeciesService birdSpeciesService; + + private final BirdMediaService birdMediaService; + + private final DeviceInfoService deviceInfoService; + + private final ProtectedAreaSpeciesService protectedAreaSpeciesService; + + private final FavoriteMediaService favoriteMediaService; + + private final SysDeptMapper sysDeptMapper; + + @Resource + private final EsResultRecordDetailsMapper esResultRecordDetailsMapper; + + @Resource + private final ResultRecordDetailsMapper resultRecordDetailsMapper; + + private final String indexName = "animal_record_details"; + + + @Override + public TableDataInfo querySpeciesRecordListByParam(JSONObject param, int type) { +// Long deptId = param.getLong("deptId"); +// if (deptId != null) { +// List deviceIds = deviceInfoService.getDeviceIdListByDeptId(deptId); +// if (!deviceIds.isEmpty()) { +// param.put("deviceIds", deviceIds); +// } else { +// return null; +// } +// } +// boolean check = getDeviceIdsByParam(param); +// if (!check) { +// return null; +// } + Long deptId = param.getLong("deptId"); + if (deptId != null) { + List deptIds = sysDeptMapper.getDeptSubIdsById(deptId); + List ids = deviceInfoService.getDeviceIdsInList(deptIds); + if (ids != null && !ids.isEmpty()) { + param.put("deviceIds", ids); + } else { + return new TableDataInfo(); + } + } + + Long speciesId = param.getLong("speciesId"); + if (speciesId != null) { + param.remove("excludeIds"); + param.remove("ids"); + } + + String strIds = param.getString("strIds"); + if (strIds != null && !strIds.isEmpty()) { + String[] array = strIds.split("_"); + JSONArray jsonArray = new JSONArray(); + for (String str : array) { + jsonArray.add(Long.parseLong(str)); + } + param.put("inDocIds", jsonArray); + } + String excludeIds = param.getString("excludeIds"); + if (excludeIds != null && !excludeIds.isEmpty()) { + String[] array = excludeIds.split("_"); + JSONArray ids = new JSONArray(); + for (String str : array) { + ids.add(Long.parseLong(str)); + } + if (!ids.isEmpty()) { + param.put("ids", ids); + } + } + + Long noSpeciesId = param.getLong("noSpeciesId"); + if (noSpeciesId != null) { + JSONArray ids = new JSONArray(); + ids.add(noSpeciesId); + param.put("ids", ids); + } +// log.info("============={}", param); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + Integer pageNum = param.getInteger("pageNum"); + if (pageNum == null) { + pageNum = 1; + } + Integer pageSize = param.getInteger("pageSize"); + if (pageSize == null) { + pageSize = 10; + } + + SortOrder sortOrder; + String dateFrom = param.getString("dateFrom"); + if (dateFrom != null && !dateFrom.isEmpty()) { + sortOrder = SortOrder.Asc; + } else { + sortOrder = SortOrder.Desc; + } + + +// log.info("=================1===================="); + int pageFrom = (pageNum - 1) * pageSize; + try { + Integer finalPageSize = pageSize; + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(boolQueryBuilder.build())).sort(sort -> sort.field(f -> f.field("record_time").order(sortOrder))) +// .source(sc -> sc +// .filter(f -> f +// .includes(Arrays.asList(sources)) +// ) +// ) + .from(pageFrom).size(finalPageSize), EsResultRecordDetails.class); + + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } + +// if (type == 1) { +// list = list.stream().peek(item -> { +// item.setBirdIcon(birdMediaService.getBirdIconImageBySpeciesId(item.getSpeciesId())); +// }).collect(Collectors.toList()); +// } + long total = response.hits().total() != null ? response.hits().total().value() : 0; + return getTableDataInfo(list, total); + } catch (IOException e) { + logger.error(e.getMessage()); +// e.printStackTrace(); + } + return null; + } + + private ResultMedia getLastMediaInfoByParam(JSONObject param) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + try { + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(boolQueryBuilder.build())).sort(sort -> sort.field(f -> f.field("record_time").order(SortOrder.Desc))).from(0).size(1), EsResultRecordDetails.class); + + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } +// log.info("==============={}", list); + if (!list.isEmpty()) { + ResultMedia data = new ResultMedia(); + if (list.get(0).getRecordTime() != null) { + data.setTime(Date.from(list.get(0).getRecordTime().toInstant())); + } + return data; + } + } catch (IOException e) { + logger.error(e.getMessage()); +// e.printStackTrace(); + } + return null; + } + + + @Override + public TableDataInfo getProtectedAreaSpeciesByQuery(JSONObject param) { + Map map = new HashMap<>(); + Integer pageNum = param.getInteger("pageNum"); + Integer pageSize = param.getInteger("pageSize"); + PageUtils.startPage(pageNum, pageSize); + String name = param.getString("name"); + map.put("name", name); + List birdSpecies = birdSpeciesService.getBirdSpeciesListByQuery(map); + TableDataInfo rspData = new TableDataInfo(); + rspData.setCode(HttpStatus.SUCCESS); + rspData.setMsg("查询成功"); + rspData.setTotal(new PageInfo(birdSpecies).getTotal()); +// Long areaId = param.getLong("areaId"); + + List list = new ArrayList<>(); + for (BirdSpecies item : birdSpecies) { + ProtectedAreaSpecies data = new ProtectedAreaSpecies(); + data.setSpeciesId(item.getId()); + data.setBirdSpecies(item); + param.put("speciesId", item.getId()); + try { + EsResultRecordDetails resultRecordDetails = CommonEsUtils.getResultRecordDetailsByParam(param, indexName, elasticsearchClient); + if (resultRecordDetails != null) { +// data.setAreaName(resultRecordDetails.getAreaName()); + if (resultRecordDetails.getRecordTime() != null) { + data.setUpdateTime(Date.from(resultRecordDetails.getRecordTime().toInstant())); + } + } else { + data.setAreaName("当前未监测到"); + } + list.add(data); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + } + rspData.setRows(list); + return rspData; + } + + @Override + public TableDataInfo getResultMediaListByParam(JSONObject jsonObject) { + Long mediaId = jsonObject.getLong("mediaId"); + String[] sources = {"id", "record_time", "species_name", "species_id", "media_id", "media_url", "meta_url", "media_type", "favorite"}; + BoolQuery.Builder boolQueryBuilder = null; + Integer pageNum = null; + Integer pageSize = null; + + if (mediaId != null) { + JSONObject param = new JSONObject(); + param.put("mediaId", mediaId); + pageNum = 1; + pageSize = 1; + boolQueryBuilder = CommonEsUtils.createQuery(param); + } else { + pageNum = jsonObject.getInteger("pageNum"); + if (pageNum == null) { + pageNum = 1; + } + pageSize = jsonObject.getInteger("pageSize"); + if (pageSize == null) { + pageSize = 10; + } + boolQueryBuilder = CommonEsUtils.createQuery(jsonObject); + } + int pageFrom = (pageNum - 1) * pageSize; +// log.info("==================={}", pageFrom); + try { + BoolQuery.Builder finalBoolQueryBuilder = boolQueryBuilder; + Integer finalPageSize = pageSize; + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(finalBoolQueryBuilder.build())).sort(sort -> sort.field(f -> f.field("record_time").order(SortOrder.Desc)))// + .source(sc -> sc.filter(f -> f.includes(Arrays.asList(sources)))).from(pageFrom).size(finalPageSize), EsResultRecordDetails.class); + List> hitList = response.hits().hits(); + long total = response.hits().total() != null ? response.hits().total().value() : 0; + List list = new ArrayList<>(); + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } + return getTableDataInfo(list, total); + } catch (IOException ex) { +// ex.printStackTrace(); + logger.error(ex.getMessage()); + } + return getTableDataInfo(new ArrayList<>(), 0); + } + + @Override + public int deleteDocumentById(String ids) { + try { + String[] arr = ids.split(","); + for (String id : arr) { + elasticsearchClient.delete(DeleteRequest.of(d -> d.index(indexName).id(id))); + } +// DeleteResponse deleteResponse = + +// System.out.println("Document deleted with result: " + deleteResponse.result()); + return 1; + } catch (Exception ex) { + logger.error(ex.getMessage()); + } + return 0; + } + + @Override + public void bulkAddResultRecordDocument() { + int maxValue = getMaxResultRecordId(); + Map param = new HashMap<>(); + param.put("recordId", maxValue); +// log.info("=============maxId=={}", maxValue); + List list = esResultRecordDetailsMapper.getResultRecordListForElastic(param); +// log.info("=============list size=={}", list.size()); + if (!list.isEmpty()) { + List operations = new ArrayList<>(); + list.forEach(data -> { + operations.add(new BulkOperation.Builder().index(new IndexOperation.Builder().index(indexName).id(String.valueOf(data.getId())).document(data).build()).build()); + }); + try { + BulkRequest bulkRequest = new BulkRequest.Builder().operations(operations).build(); + BulkResponse bulkResponse = elasticsearchClient.bulk(bulkRequest); +// log.info("================={}", bulkResponse); + } catch (IOException ex) { +// ex.printStackTrace(); + logger.error(ex.getMessage()); + } + } + } + + @Override + public int getMaxResultRecordId() { + try { + SearchResponse searchResponse = elasticsearchClient.search(s -> s.index(indexName).size(0) + .aggregations("max_value", a -> a.max(m -> m.field("record_id"))), Void.class); + + Map aggregations = searchResponse.aggregations(); + int maxValue = 0; + + if (aggregations != null && aggregations.containsKey("max_value")) { + Aggregate maxAgg = aggregations.get("max_value"); + maxValue = (int) maxAgg.max().value(); + } + + return maxValue; + } catch (IOException ex) { +// ex.printStackTrace(); + logger.error(ex.getMessage()); + } + return 0; + } + + @Override + public boolean updateResultRecordInfoById(JSONObject object) { + return updateResultRecordEsDocument(object); + } + + private boolean updateResultRecordEsDocument(JSONObject object) { + Map params = new HashMap<>(); + String docId = object.getString("id"); + Integer checked = object.getInteger("checked"); + if (checked != null) { + params.put("checked", checked); + } + + Long speciesId = object.getLong("speciesId"); + if (speciesId != null) { + params.put("species_id", speciesId); + } + + Integer favorite = object.getInteger("favorite"); + if (favorite != null) { + params.put("favorite", favorite); + } + + String scientificName = object.getString("scientificName"); + if (scientificName != null) { + params.put("scientific_name", scientificName); + } + + Integer count = object.getInteger("count"); + if (favorite != null) { + params.put("count", count); + } + + String speciesName = object.getString("speciesName"); + if (speciesName != null) { + params.put("species_name", speciesName); + } + + Integer isDeleted = object.getInteger("isDeleted"); + if (isDeleted != null) { + params.put("is_deleted", isDeleted); + } + + String tag = object.getString("tag"); + if (tag != null && !tag.isEmpty()) { + params.put("tag", tag); + } + + String memo = object.getString("memo"); + if (memo != null && !memo.isEmpty()) { + params.put("memo", memo); + } + + UpdateRequest> request = UpdateRequest.of(builder -> builder.index(indexName).id(docId).doc(params)); + try { + elasticsearchClient.update(request, Map.class); + return true; + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return false; + } + + @Override + public boolean updateResultRecordInfoByIds(JSONObject object) { + String ids = object.getString("location"); +// System.out.println("===================" + ids); + String[] arr = ids.split(","); + for (String detailsId : arr) { + object.put("detailsId", detailsId); + updateResultRecordEsDocument(object); + } + return true; + } + + @Override + public JSONObject getBirdAudioDailyStatListByParam(JSONObject param) { + return getDailyAggregateListByParam(param, 0); + } + + + private JSONObject getDailyAggregateListByParam(JSONObject param, int type) { + JSONObject result = new JSONObject(); + try { + JSONArray jsonArray = CommonEsUtils.getSpeciesAggregateList(param, indexName, elasticsearchClient, CalendarInterval.Hour); + Map map = new HashMap<>(); + int[] hourAmount = new int[24]; + for (Object obj : jsonArray) { + JSONObject item = (JSONObject) obj; + int hour = item.getInteger("hour"); + hourAmount[hour] += item.getInteger("count"); + Long speciesId = item.getLong("speciesId"); + JSONObject objItem = map.get(speciesId); + int data1 = 0; + int data2 = 0; + if (type == 0) { + if (objItem == null) { + objItem = new JSONObject(); + } else { + data1 = objItem.getInteger("data1"); + data2 = objItem.getInteger("data2"); + } + hour = item.getInteger("hour"); + if (hour < 18 && hour >= 6) { + data1 += item.getInteger("count"); + } else { + data2 += item.getInteger("count"); + } + } else if (type == 1) { + if (objItem == null) { + objItem = new JSONObject(); + } else { + data1 = objItem.getInteger("data1"); + data2 = objItem.getInteger("data2"); + } + hour = item.getInteger("hour"); + int count = item.getInteger("count"); + if (hour <= 12) { + if (count > data1) { + data1 = count; + } + } else { + if (count > data2) { + data2 = count; + } + } + } + objItem.put("id", speciesId); + objItem.put("data1", data1); + objItem.put("data2", data2); + objItem.put("dataAll", data1 + data2); + map.put(speciesId, objItem); + } + result.put("hourAmount", hourAmount); + Object[] array = map.values().toArray(); + JSONArray list = new JSONArray(); + for (Object obj : array) { + JSONObject objItem = (JSONObject) obj; + Long id = objItem.getLong("id"); + BirdSpecies birdSpecies = birdSpeciesService.selectBirdSpeciesForEsById(id); + objItem.put("name", birdSpecies.getName()); + objItem.put("familyId", birdSpecies.getFamilyId()); + objItem.put("familyName", birdSpecies.getFamilyName()); + objItem.put("code", birdSpecies.getSpeciesType()); + objItem.put("codeName", birdSpecies.getSpeciesTypeName()); + list.add(objItem); + } + list.sort(Comparator.comparing(obj -> ((JSONObject) obj).getInteger("dataAll")).reversed()); + result.put("list", list); + } catch (IOException ex) { +// ex.printStackTrace(); + logger.error(ex.getMessage()); + } + return result; + } + + @Override + public JSONObject getBirdDailyStatListByParam(JSONObject param) { +// param.put("recordType", 1); + JSONObject object = getDailyAggregateListByParam(param, 1); + JSONArray array = object.getJSONArray("list"); + if (array != null && !array.isEmpty()) { + JSONArray ids = new JSONArray(); + for (Object obj : array) { + JSONObject jsonObject = (JSONObject) obj; + ids.add(jsonObject.getLong("id")); + } + param.put("ids", ids); + } else { + array = new JSONArray(); + } + param.put("recordType", 2); + JSONObject subObj = getDailyAggregateListByParam(param, 1); + if (subObj.getJSONArray("list") != null) { + array.addAll(subObj.getJSONArray("list")); + } + array.sort(Comparator.comparing(obj -> ((JSONObject) obj).getInteger("dataAll")).reversed()); + object.put("list", array); + return object; + } + + @Override + public JSONObject getBirdAudioMonthStatListByParam(JSONObject param) { + JSONObject result = new JSONObject(); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0) + .query(q -> q.bool(boolQueryBuilder.build())) + .aggregations("by_month", a -> a.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time").calendarInterval(CalendarInterval.Month))) + .aggregations("by_hour", b -> b.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time").calendarInterval(CalendarInterval.Hour))) + .aggregations("sum_count_field", s -> s.sum(SumAggregation.of(sa -> sa.field("count"))))))); + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate byMonthAgg = aggs.get("by_month"); + JSONArray jsonArray = new JSONArray(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + + byMonthAgg.dateHistogram().buckets().array().forEach(monthBucket -> { + String month = monthBucket.keyAsString(); + Aggregate byHourAgg = monthBucket.aggregations().get("by_hour"); + byHourAgg.dateHistogram().buckets().array().forEach(hourBucket -> { + String hour = hourBucket.keyAsString(); + if (month != null && hour != null) { + JSONObject item = new JSONObject(); + LocalDateTime dateTime = LocalDateTime.parse(month, formatter); + item.put("month", dateTime.getMonth().getValue()); + dateTime = LocalDateTime.parse(hour, formatter); + item.put("hour", dateTime.getHour()); + Aggregate sumAgg = hourBucket.aggregations().get("sum_count_field"); + item.put("count", (int) sumAgg.sum().value()); + jsonArray.add(item); + } + }); + }); + Map map = new HashMap<>(); + for (Object obj : jsonArray) { + JSONObject item = (JSONObject) obj; + + Integer month = item.getInteger("month"); + JSONObject objItem = map.get(month); + int data1 = 0; + int data2 = 0; + + if (objItem == null) { + objItem = new JSONObject(); + } else { + data1 = objItem.getInteger("data1"); + data2 = objItem.getInteger("data2"); + } + Integer hour = item.getInteger("hour"); + if (hour < 18 && hour >= 6) { + data1 += item.getInteger("count"); + } else { + data2 += item.getInteger("count"); + } + + objItem.put("month", month); + objItem.put("data1", data1); + objItem.put("data2", data2); + objItem.put("dataAll", data1 + data2); + map.put(month, objItem); + } + Object[] array = map.values().toArray(); + JSONArray list = new JSONArray(); + for (Object obj : array) { + JSONObject objItem = (JSONObject) obj; + list.add(objItem); + } + list.sort(Comparator.comparing(obj -> ((JSONObject) obj).getInteger("month"))); + result.put("list", array); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return result; + } + + @Override + public JSONObject getBirdAudioYearStatListByParam(JSONObject param) { + JSONObject result = new JSONObject(); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0).query(q -> q.bool(boolQueryBuilder.build()))// + .aggregations("by_year", a -> a.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time").calendarInterval(CalendarInterval.Year)))// + .aggregations("by_month", b -> b.dateHistogram(DateHistogramAggregation.of(dh -> dh.field("record_time").calendarInterval(CalendarInterval.Month)))// + .aggregations("sum_count_field", s -> s.sum(SumAggregation.of(sa -> sa.field("count")))))));// + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate byYearAgg = aggs.get("by_year"); + JSONArray jsonArray = new JSONArray(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + + byYearAgg.dateHistogram().buckets().array().forEach(yearBucket -> { + String year = yearBucket.keyAsString(); + Aggregate byMonthAgg = yearBucket.aggregations().get("by_month"); + byMonthAgg.dateHistogram().buckets().array().forEach(hourBucket -> { + String month = hourBucket.keyAsString(); + if (year != null && month != null) { + JSONObject item = new JSONObject(); + LocalDateTime dateTime = LocalDateTime.parse(year, formatter); + item.put("year", dateTime.getYear()); + dateTime = LocalDateTime.parse(month, formatter); + item.put("month", dateTime.getMonth().getValue()); + Aggregate sumAgg = hourBucket.aggregations().get("sum_count_field"); + item.put("count", (int) sumAgg.sum().value()); + jsonArray.add(item); + } + }); + }); + Map map = new HashMap<>(); + for (Object obj : jsonArray) { + JSONObject item = (JSONObject) obj; + + Integer year = item.getInteger("year"); + JSONObject objItem = map.get(year); + int data1 = 0; + int data2 = 0; + int data3 = 0; + int data4 = 0; + if (objItem == null) { + objItem = new JSONObject(); + } else { + data1 = objItem.getInteger("data1"); + data2 = objItem.getInteger("data2"); + data3 = objItem.getInteger("data3"); + data4 = objItem.getInteger("data4"); + } + Integer month = item.getInteger("month"); + if (month < 4 && month >= 1) { + data1 += item.getInteger("count"); + } else if (month < 7 && month >= 4) { + data2 += item.getInteger("count"); + } else if (month < 10 && month >= 7) { + data3 += item.getInteger("count"); + } else { + data4 += item.getInteger("count"); + } + + objItem.put("year", year); + objItem.put("data1", data1); + objItem.put("data2", data2); + objItem.put("data3", data3); + objItem.put("data4", data4); + map.put(year, objItem); + } + Object[] array = map.values().toArray(); + JSONArray list = new JSONArray(); + for (Object obj : array) { + JSONObject objItem = (JSONObject) obj; + list.add(objItem); + } + list.sort(Comparator.comparing(obj -> ((JSONObject) obj).getInteger("year"))); + result.put("list", array); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return result; + } + + @Override + public List getDeviceInfoListForPage(DeviceInfo deviceInfo) { + List list = deviceInfoService.getDeviceInfoListForPage(deviceInfo); + JSONObject param = new JSONObject(); + list.forEach(item -> { + param.put("deviceId", item.getId()); + if (Objects.equals(item.getDeviceType(), "1")) { + param.put("mediaType", 3); + } else { + param.put("mediaType", 1); + } + item.setResultMedia(getLastMediaInfoByParam(param)); + }); + return list; + } + + @Override + public JSONArray getSpeciesListForSelect() { + JSONArray array = new JSONArray(); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName) // + .size(0) // + .query(q -> q.matchAll(MatchAllQuery.of(m -> m)))// + .aggregations("by_species_id", a -> a.terms(TermsAggregation.of(dh -> dh.field("species_id").size(10000)))) + ); + try { + List list = new ArrayList<>(); + // Execute the search request + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate bySpeciesIdAgg = aggs.get("by_species_id"); + bySpeciesIdAgg.lterms().buckets().array().forEach(idBucket -> { + list.add(idBucket.key()); + }); + Map map = new HashMap<>(); + map.put("list", list); + List birdSpecies = birdSpeciesService.getSpeciesSelectListByParam(map); + birdSpecies.forEach(item -> { + JSONObject obj = new JSONObject(); + obj.put("speciesId", item.getId()); + obj.put("speciesName", item.getName()); + array.add(obj); + }); + return array; + } catch (IOException ex) { + logger.error(ex.getMessage()); +// ex.printStackTrace(); + } + + return null; + } + + @Override + public JSONObject getResultMediaStatByParam() { + JSONObject result = new JSONObject(); + JSONObject param = new JSONObject(); + if (SecurityUtils.getDeptId() != 1) { + List list = deviceInfoService.getPostListByUserId(SecurityUtils.getUserId()); + param.put("deviceIds", JSONArray.parseArray(JSONArray.toJSONString(list))); + } + param.put("mediaType", 2); + result.put("imageAmount", getSpeciesAmountByParam(param)); + param.put("mediaType", 3); + result.put("videoAmount", getSpeciesAmountByParam(param)); + return result; + } + + private long getSpeciesAmountByParam(JSONObject param) { + try { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + CountRequest countRequest = CountRequest.of(r -> r + .index(indexName) + .query(q -> q.bool(boolQueryBuilder.build())) + ); + CountResponse countResponse = elasticsearchClient.count(countRequest); + return countResponse.count(); + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return 0; + } + + @Override + public JSONObject getSpeciesStatListByParam(JSONObject param) { + JSONObject result = new JSONObject(); + Map map = new HashMap<>(); + Integer limit = param.getInteger("limit"); + map.put("limit", limit); + List list = protectedAreaSpeciesService.getProtectedAreaSpeciesListByAreaId(map); + if (list == null) { + return result; + } + param.clear(); + if (SecurityUtils.getDeptId() != 1) { + List ids = deviceInfoService.getPostListByUserId(SecurityUtils.getUserId()); + param.put("deviceIds", JSONArray.parseArray(JSONArray.toJSONString(ids))); + } + list.forEach(item -> { + param.put("speciesId", item.getSpeciesId()); + param.put("mediaType", 3); + BirdStatDataVo data = getSpeciesStatInfoByParam(param); + List ids = new ArrayList<>(); + if (data != null) { + item.setVideoCount(data.getCount()); + ids.addAll(data.getDeviceIds()); + } + param.put("mediaType", 2); + data = getSpeciesStatInfoByParam(param); + if (data != null) { + item.setImageCount(data.getCount()); + ids.addAll(data.getDeviceIds()); + } + if (!ids.isEmpty()) { + Set idsSet = new HashSet<>(ids); + item.setDeviceIds(new ArrayList<>(idsSet)); + } + }); + result.put("list", list); + return result; + } + + @Override + public List getDeviceInfoListForGisMap() { + List list = deviceInfoService.getDeviceInfoListForGisMap(); + JSONObject param = new JSONObject(); + list.forEach(item -> { + param.put("deviceId", item.getId()); + item.setRecordCount(getSpeciesAmountByParam(param)); + }); + return list; + } + + @Override + public JSONObject getSpeciesStatInfo(JSONObject param) { + JSONObject result = new JSONObject(); +// boolean check = getDeviceIdsByParam(param); + Long deptId = param.getLong("deptId"); + if (deptId != null) { + List deptIds = sysDeptMapper.getDeptSubIdsById(deptId); + List ids = deviceInfoService.getDeviceIdsInList(deptIds); + if (ids != null && !ids.isEmpty()) { + param.put("deviceIds", ids); + } else { + return result; + } + } + Long speciesId = param.getLong("speciesId"); + if (speciesId != null) { + param.remove("excludeIds"); + } + param.put("isDeleted", 0); + param.put("favorite", 1); + result.put("favoriteCount", getSpeciesStatAmount(param)); + param.remove("favorite", 1); +// log.info("=========1======={}=", param); +// param.remove("speciesId"); + result.put("allAmount", getSpeciesStatAmount(param)); + if (speciesId == null) { + param.put("speciesIds", new Integer[]{0, 1544}); + result.put("emptyAmount", getSpeciesStatAmount(param)); + } else { + result.put("emptyAmount", 0); + } +// log.info("=========2======={}=", param); + + param.remove("speciesIds"); + param.put("checked", 0); + result.put("uncheckAmount", getSpeciesStatAmount(param)); +// if (check) { +// result.put("favoriteList", getFavoriteList(param)); +// param.put("favorite", 1); +// result.put("favoriteCount", getSpeciesStatAmount(param)); +// param.remove("favorite", 1); +// param.remove("speciesId"); +// result.put("allAmount", getSpeciesStatAmount(param)); +// param.put("speciesId", 0); +// result.put("emptyAmount", getSpeciesStatAmount(param)); +// param.remove("speciesId"); +// param.put("checked", 0); +// result.put("uncheckAmount", getSpeciesStatAmount(param)); +// result.put("speciesList", getSpeciesAmountListByParam(param)); +// } + return result; + } + + @Override + public boolean updateSpeciesInfo(JSONObject object) { + ResultRecordDetails resultRecordDetails = new ResultRecordDetails(); + // + Map params = new HashMap<>(); + String docId = object.getString("id"); + Long detailsId = object.getLong("detailsId"); + if (detailsId != null) { + params.put("detailsId", JsonData.of(detailsId)); + resultRecordDetails.setDetailsId(detailsId); + } + + Long speciesId = object.getLong("speciesId"); + if (speciesId != null) { + params.put("speciesId", JsonData.of(speciesId)); + resultRecordDetails.setSpeciesId(speciesId); + } + + String speciesName = object.getString("speciesName"); + if (speciesName != null && !speciesName.isEmpty()) { + params.put("speciesName", JsonData.of(speciesName)); + } + String scientificName = object.getString("scientificName"); + if (scientificName != null && !scientificName.isEmpty()) { + params.put("scientificName", JsonData.of(scientificName)); + } + Integer count = object.getInteger("count"); + if (count != null) { + params.put("count", JsonData.of(count)); + resultRecordDetails.setCount(count); + } else { + params.put("count", JsonData.of(1)); + resultRecordDetails.setCount(1); + } + resultRecordDetailsMapper.updateResultRecordDetails(resultRecordDetails); + Script script = new Script.Builder() + .inline(s -> s + .lang(ScriptLanguage.Painless) + .source("for (item in ctx._source.items) { if (item.details_id == params.detailsId) { item.species_id = params.speciesId;item.species_name = params.speciesName;item.scientific_name = params.scientificName;item.count = params.count; } }") + .params(params) + ) + .build(); + UpdateRequest request = new UpdateRequest.Builder<>() + .index(indexName) + .id(docId) + .script(script) + .build(); + try { + UpdateResponse response = elasticsearchClient.update(request, Object.class); + } catch (Exception ex) { + return false; + } + return true; + } + + @Override + public boolean deleteSpeciesInfo(JSONObject object) { + Map params = new HashMap<>(); + String docId = object.getString("id"); + Long detailsId = object.getLong("detailsId"); + if (detailsId != null) { + params.put("detailsId", JsonData.of(detailsId)); + + } + resultRecordDetailsMapper.deleteResultRecordDetailsByDetailsId(detailsId); + Script script = new Script.Builder() + .inline(s -> s + .lang(ScriptLanguage.Painless) + .source("ctx._source.items.removeIf(item -> item.details_id == params.detailsId)") + .params(params) + ) + .build(); + UpdateRequest request = new UpdateRequest.Builder<>() + .index(indexName) + .id(docId) + .script(script) + .build(); + try { + UpdateResponse response = elasticsearchClient.update(request, Object.class); + } catch (Exception ex) { + return false; + } + return true; + } + + @Override + public Long addSpeciesInfo(JSONObject object) { + Map params = new HashMap<>(); + Map newItem = new HashMap<>(); + String docId = object.getString("id"); + ResultRecordDetails resultRecordDetails = new ResultRecordDetails(); + + Long speciesId = object.getLong("speciesId"); + newItem.put("species_id", speciesId); + resultRecordDetails.setSpeciesId(speciesId); + + String speciesName = object.getString("speciesName"); + newItem.put("species_name", speciesName); + + String scientificName = object.getString("scientificName"); + newItem.put("scientific_name", scientificName); +// Long recordId = object.getLong("recordId"); + resultRecordDetails.setRecordId(0L); + newItem.put("record_id", 0); + + resultRecordDetails.setHabitat("0"); + resultRecordDetails.setMediaId(Long.parseLong(docId)); + + Integer count = object.getInteger("count"); + if (count != null) { + newItem.put("count", count); + resultRecordDetails.setCount(count); + } else { + newItem.put("count", 1); + resultRecordDetails.setCount(1); + } + + resultRecordDetailsMapper.insertResultRecordDetails(resultRecordDetails); + newItem.put("details_id", resultRecordDetails.getDetailsId()); + params.put("newItem", JsonData.of(newItem)); + + Script script = new Script.Builder() + .inline(s -> s + .lang(ScriptLanguage.Painless) + .source("if (ctx._source.items == null) { ctx._source.items = []; } ctx._source.items.add(params.newItem);") + .params(params) + ) + .build(); + UpdateRequest request = new UpdateRequest.Builder<>() + .index(indexName) + .id(docId) + .script(script) + .build(); + try { + UpdateResponse response = elasticsearchClient.update(request, Object.class); + } catch (Exception ex) { + return 0L; + } + return resultRecordDetails.getDetailsId(); + } + + @Override + public List getFavoriteMediaList(JSONObject param) { + if (param == null) { + param = new JSONObject(); + } + param.put("isDeleted", 0); + param.put("mediaType", 3); + param.put("favorite", 1); + String[] sources = {"id", "record_time", "species_name", "species_id", "media_id", "media_url", "meta_url", "media_type", "favorite"}; + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + try { + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(boolQueryBuilder.build())).sort(sort -> sort.field(f -> f.field("record_time").order(SortOrder.Desc)))// + .source(sc -> sc.filter(f -> f.includes(Arrays.asList(sources)))), EsResultRecordDetails.class); + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } + return list; + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return null; + } + + @Override + public List getMediaListByItemId(JSONObject param) { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + try { + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(boolQueryBuilder.build())), EsResultRecordDetails.class); + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } + return list; + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return null; + } + + private long getSpeciesStatAmount(JSONObject param) { +// log.info("=======2============={}", param); +// List deviceIds = getDeviceIdsByParam(param); +// log.info("=======3============={}", deviceIds); +// if (deviceIds != null && !deviceIds.isEmpty()) { +// param.put("deviceIds", deviceIds); +// } +// if (type == 0) { +// param.put("speciesId", 0); +// } + + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + CountRequest countRequest = CountRequest.of(builder -> builder.index(indexName).query(q -> q.bool(boolQueryBuilder.build()))); + try { + CountResponse response = elasticsearchClient.count(countRequest); + return response.count(); + } catch (Exception ex) { + logger.error("error", ex); + } + return 0; + } + + private JSONArray getSpeciesAmountListByParam(JSONObject param) { +// param.put("isDeleted", 0); +// List deviceIds = getDeviceIdsByParam(param); +// if (deviceIds != null && !deviceIds.isEmpty()) { +// param.put("deviceIds", deviceIds); +// } + JSONArray ids = new JSONArray(); + ids.add(0L); + param.put("ids", ids); + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); + SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(indexName).size(0).query(q -> q.bool(boolQueryBuilder.build()))// + .aggregations("by_speciesId", a -> a.terms(TermsAggregation.of(ta -> ta.field("species_id").size(10000)))// + ));// + try { + SearchResponse searchResponse = elasticsearchClient.search(searchRequest, Void.class); + Map aggs = searchResponse.aggregations(); + Aggregate bySpeciesAgg = aggs.get("by_speciesId"); + JSONArray jsonArray = new JSONArray(); + bySpeciesAgg.lterms().buckets().array().forEach(speciesBucket -> { + JSONObject object = new JSONObject(); + Long speciesId = speciesBucket.key(); + object.put("speciesId", speciesId); + object.put("amount", speciesBucket.docCount()); + object.put("speciesName", birdSpeciesService.getSpeciesNameById(speciesId)); + jsonArray.add(object); + }); + return jsonArray; + } catch (Exception ex) { + logger.error("error", ex); + } + return null; + } + + + private boolean getDeviceIdsByParam(JSONObject param) { + Long deptId = param.getLong("deptId"); + if (deptId != null) { + List ids = deviceInfoService.getDeviceIdListByDeptId(deptId); + if (ids == null || ids.isEmpty()) { + return false; + } else { + param.put("deviceIds", ids); + return true; + } + } else { + if (SecurityUtils.getUserId() != 1L && SecurityUtils.getDeptId() == 0) { + List list = deviceInfoService.getDeviceStatInfoByParam(); + if (list == null || list.isEmpty()) { + return false; + } else { + List ids = list.stream().map(DeviceInfo::getId).collect(Collectors.toList()); + param.put("deviceIds", ids); + return true; + } + } else { + return true; + } + + } + } + + private List getFavoriteList(JSONObject param) { + Map map = new HashMap<>(); +// List deviceIds = getDeviceIdsByParam(param); +// if (deviceIds == null || deviceIds.isEmpty()) { +// return null; +// } else { +// +// } + map.put("list", param.getJSONArray("deviceIds")); + return favoriteMediaService.getFavoriteMediaListByParam(map); + } + + + private BirdStatDataVo getSpeciesStatInfoByParam(JSONObject param) { + try { + BoolQuery.Builder boolQueryBuilder = CommonEsUtils.createQuery(param); +// String[] sources = {"device_id"}; + SearchResponse response = elasticsearchClient.search(s -> s.trackTotalHits(new TrackHits.Builder().enabled(true).build()).index(indexName)// + .query(q -> q.bool(boolQueryBuilder.build())) +// .source(sc -> sc +// .filter(f -> f +// .includes(Arrays.asList(sources)) +// ) +// ) + .from(0).size(10000), EsResultRecordDetails.class); + + List> hitList = response.hits().hits(); + List list = new ArrayList<>(); + for (Hit mapHit : hitList) { + list.add(mapHit.source()); + } + BirdStatDataVo data = new BirdStatDataVo(); + data.setCount(list.size()); + List ids = list.stream().map(EsResultRecordDetails::getDeviceId).collect(Collectors.toList()); + data.setDeviceIds(ids); + return data; + } catch (IOException ex) { + logger.error(ex.getMessage()); + } + return null; + } + + private TableDataInfo getTableDataInfo(List list, long total) { + TableDataInfo tableDataInfo = new TableDataInfo(); + tableDataInfo.setRows(list); + tableDataInfo.setTotal(total); + tableDataInfo.setCode(HttpStatus.SUCCESS); + tableDataInfo.setMsg("查询成功"); + return tableDataInfo; + } + +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/ChainLength.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/ChainLength.java new file mode 100644 index 0000000..ecf92b7 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/ChainLength.java @@ -0,0 +1,126 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.ChainLengthItem; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.util.GeoUtils; +import net.hzvtc.stat.util.StatisticsUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * R 中 chainLength(chain) / chainLengths(chainDB) 的 Java 实现。 + * + *

算法: + *

    + *
  1. 把每条足迹链转 UTM 米。
  2. + *
  3. 计算相邻点的距离(km),若 > 100km 视为跨区清零。
  4. + *
  5. 累加得到总链长(km)。
  6. + *
+ *

+ * + *

chainLengths 按 Species/Line 聚合,额外给出每物种平均步长(m)与推算步数。

+ */ +public final class ChainLength { + + private ChainLength() { + } + + /** + * 单条足迹链长度(km)。 + * + * @param chain 足迹链 + * @return 链长(km) + */ + public static double chainLength(List chain) { + if (chain == null || chain.size() < 2) { + return 0.0; + } + double[][] xy = GeoUtils.wgs84ToUtmAuto(chain, p -> p.getLon(), p -> p.getLat()); + double totalKm = 0.0; + for (int i = 1; i < xy.length; i++) { + double dx = (xy[i][0] - xy[i - 1][0]) / 1000.0; + double dy = (xy[i][1] - xy[i - 1][1]) / 1000.0; + double d = Math.sqrt(dx * dx + dy * dy); + // 跨区 (>100 km) 清零 + totalKm += d > 100.0 ? 0.0 : d; + } + return Math.round(totalKm * 1000.0) / 1000.0; + } + + /** + * 多物种多链的链长聚合(每物种一行)。 + */ + public static List chainLengths(List records) { + if (records == null || records.isEmpty()) { + return new ArrayList<>(); + } + // 按 Species 分组 + Map> bySpecies = new HashMap<>(); + for (ChainRecord r : records) { + bySpecies.computeIfAbsent(r.getSpecies(), k -> new ArrayList<>()).add(r); + } + + List result = new ArrayList<>(); + for (Map.Entry> e : bySpecies.entrySet()) { + String species = e.getKey(); + List sp = e.getValue(); + + // 按 Line 分组 + Map> byLine = new HashMap<>(); + for (ChainRecord r : sp) { + byLine.computeIfAbsent(r.getLine(), k -> new ArrayList<>()).add(r); + } + + List distances = new ArrayList<>(); + List steps = new ArrayList<>(); + for (List lineRecs : byLine.values()) { + List chain = new ArrayList<>(lineRecs.size()); + for (ChainRecord r : lineRecs) { + chain.add(new FootprintPoint(r.getLon(), r.getLat(), r.getDate())); + } + double distKm = chainLength(chain); + distances.add(distKm); + + // 平均步长(m):相邻点距离(m)求均值 + double[][] xy = GeoUtils.wgs84ToUtmAuto(chain, p -> p.getLon(), p -> p.getLat()); + if (xy.length >= 2) { + double sum = 0.0; + int cnt = 0; + for (int i = 1; i < xy.length; i++) { + double dx = xy[i][0] - xy[i - 1][0]; + double dy = xy[i][1] - xy[i - 1][1]; + double d = Math.sqrt(dx * dx + dy * dy); + if (d > 0 && d <= 100000) { + sum += d; + cnt++; + } + } + if (cnt > 0) { + steps.add(sum / cnt); + } + } + } + + double[] dArr = distances.stream().mapToDouble(Double::doubleValue).toArray(); + double[] sArr = steps.stream().mapToDouble(Double::doubleValue).toArray(); + double meanDist = StatisticsUtils.mean(dArr); + double sdDist = StatisticsUtils.sd(dArr); + double meanStep = sArr.length > 0 ? StatisticsUtils.mean(sArr) : 0.0; + long stepNo = meanStep > 0 ? (long) Math.floor(meanDist * 1000.0 / meanStep * 30.0) : 0L; + + result.add(new ChainLengthItem( + species, + Math.round(meanDist * 1000.0) / 1000.0, + Math.round(sdDist * 1000.0) / 1000.0, + Math.round(meanStep * 10.0) / 10.0, + stepNo, + dArr.length + )); + } + return result; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DailyRhythm.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DailyRhythm.java new file mode 100644 index 0000000..209e24c --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DailyRhythm.java @@ -0,0 +1,84 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.DensitySeries; +import net.hzvtc.stat.util.DensityUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * R 中 dailyRhythm(x) 的 Java 实现:返回 24 小时活动节律的两条 KDE 曲线(bw=0.3 细线 / bw=2 粗线)。 + * + *

算法: + *

    + *
  1. 把每个 Time 字符串 (HH:mm:ss) 转为相对 0 点的秒数,再除以 3600 得到 0-24 小时浮点值。
  2. + *
  3. 在 [0, 24] 区间分别用 bw=0.3 与 bw=2 计算高斯核密度。
  4. + *
  5. 返回两条 {@link DensitySeries}(前端用 ECharts 画)。
  6. + *
+ *

+ */ +public final class DailyRhythm { + + private DailyRhythm() { + } + + /** 默认采样点数(与 R density 默认 n=512 一致)。 */ + private static final int N = 512; + + /** + * @param trapresult 数据(含 Time 字段) + * @return 两条曲线:[bw=0.3, bw=2] + */ + public static List dailyRhythm(List trapresult) { + double[] hours = toHourArray(trapresult); + + DensitySeries fine = DensityUtils.gaussianKde(hours, 0.3, N, 0.0, 24.0, "bw=0.3"); + DensitySeries coarse = DensityUtils.gaussianKde(hours, 2.0, N, 0.0, 24.0, "bw=2"); + + List list = new ArrayList<>(2); + list.add(fine); + list.add(coarse); + return list; + } + + /** 仅返回单个 bw 的曲线(便于前端切换不同精度)。 */ + public static DensitySeries dailyRhythm(List trapresult, double bw) { + double[] hours = toHourArray(trapresult); + return DensityUtils.gaussianKde(hours, bw, N, 0.0, 24.0, "bw=" + bw); + } + + /** + * 把 Time 字符串解析为 0-24 的小时浮点数;不可解析的值会被跳过。 + */ + private static double[] toHourArray(List trapresult) { + List list = new ArrayList<>(); + for (CameraPoint p : trapresult) { + String t = p.getTime(); + if (t == null || t.isEmpty()) { + continue; + } + String[] parts = t.split(":"); + if (parts.length < 2) { + continue; + } + try { + int h = Integer.parseInt(parts[0]); + int m = Integer.parseInt(parts[1]); + double s = parts.length > 2 ? Double.parseDouble(parts[2]) : 0.0; + double h24 = h + m / 60.0 + s / 3600.0; + if (h24 < 0 || h24 > 24) { + continue; + } + list.add(h24); + } catch (NumberFormatException ignored) { + // 跳过无法解析的 Time + } + } + double[] arr = new double[list.size()]; + for (int i = 0; i < arr.length; i++) { + arr[i] = list.get(i); + } + return arr; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DensityEstimate.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DensityEstimate.java new file mode 100644 index 0000000..2ed29a5 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/DensityEstimate.java @@ -0,0 +1,383 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.RasterResult; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +/** + * 种群密度估计:基于随机森林的物种分布模型(SDM),输出栅格化出现概率图。 + * + *

结构对应"随机森林 SDM"的通用流程(cameratrapR 之外的扩展分析): + *

    + *
  1. 按相机点位聚合照片总数(红外相机点位 + 积累照片总数作为出现强度)。
  2. + *
  3. 以点位分布范围为边界(外扩 3 格)建立规则网格。
  4. + *
  5. 每格特征:经度、纬度、高斯核照片密度(KDE)、到最近相机距离。
  6. + *
  7. 训练样本:相机所在格 = 出现(按照片数加权复制);远离相机的背景格 = 未出现(随机抽样)。
  8. + *
  9. Bootstrap 随机森林分类器(随机特征子集 + Gini 分裂,叶结点存正样比例), + * 对每个格子预测出现概率,值域 [0, 1]。
  10. + *
+ * 每个物种的照片分布不同,因此每个物种得到各自的密度栅格。

+ */ +public final class DensityEstimate { + + private DensityEstimate() { + } + + /** 栅格总格数上限(超过则自动放粗格宽)。 */ + private static final int MAX_CELLS = 200000; + + /** KDE 带宽(格宽倍数)。 */ + private static final double BANDWIDTH_CELLS = 3.0; + + /** 每个出现格按照片数复制的次数上限。 */ + private static final int MAX_WEIGHT = 10; + + /** 背景未出现格数量(出现样本数的倍数)。 */ + private static final int BACKGROUND_RATIO = 3; + + /** + * @param data 相机记录(lon/lat/groupSize/species) + * @param cellSize 格宽(度),≤0 时自动按范围选择 + * @param nTree 随机森林树数 + * @param seed 随机种子 + * @return 栅格化出现概率 + */ + public static RasterResult densityEstimate(List data, double cellSize, + int nTree, long seed) { + if (data == null || data.isEmpty()) { + return new RasterResult(0, 0, 0, 0, 0, new double[0], 0, 0, 0); + } + Random rng = new Random(seed); + + // 1) 聚合相机点位:key(lon,lat) → 照片总数 + Map cameras = new HashMap<>(); + long photoTotal = 0; + for (CameraPoint p : data) { + String key = p.getLon() + "," + p.getLat(); + double[] agg = cameras.get(key); + if (agg == null) { + cameras.put(key, new double[]{p.getLon(), p.getLat(), Math.max(1, p.getGroupSize())}); + } else { + agg[2] += Math.max(1, p.getGroupSize()); + } + photoTotal += Math.max(1, p.getGroupSize()); + } + + // 2) 边界 + 网格(外扩 3 格) + double minLon = Double.POSITIVE_INFINITY, maxLon = Double.NEGATIVE_INFINITY; + double minLat = Double.POSITIVE_INFINITY, maxLat = Double.NEGATIVE_INFINITY; + for (double[] c : cameras.values()) { + minLon = Math.min(minLon, c[0]); + maxLon = Math.max(maxLon, c[0]); + minLat = Math.min(minLat, c[1]); + maxLat = Math.max(maxLat, c[1]); + } + if (cellSize <= 0) { + cellSize = Math.max(maxLon - minLon, maxLat - minLat) / 99.0; + } + double span = Math.max(Math.max(maxLon - minLon, maxLat - minLat), cellSize); + while (Math.ceil((span + 6 * cellSize) / cellSize) + * Math.ceil((span + 6 * cellSize) / cellSize) > MAX_CELLS) { + cellSize *= 1.5; + } + double pad = 3 * cellSize; + double x0 = Math.floor((minLon - pad) / cellSize) * cellSize; + double y0 = Math.floor((minLat - pad) / cellSize) * cellSize; + double x1 = Math.ceil((maxLon + pad) / cellSize) * cellSize; + double y1 = Math.ceil((maxLat + pad) / cellSize) * cellSize; + int nx = Math.max(2, (int) Math.round((x1 - x0) / cellSize)); + int ny = Math.max(2, (int) Math.round((y1 - y0) / cellSize)); + + // 3) 相机 → 格索引(去重后同格合并权重) + Map presenceWeight = new HashMap<>(); + double[] camLon = new double[cameras.size()]; + double[] camLat = new double[cameras.size()]; + double[] camCount = new double[cameras.size()]; + int ci = 0; + for (double[] c : cameras.values()) { + camLon[ci] = c[0]; + camLat[ci] = c[1]; + camCount[ci] = c[2]; + ci++; + int col = clamp((int) Math.floor((c[0] - x0) / cellSize), 0, nx - 1); + int row = clamp((int) Math.floor((c[1] - y0) / cellSize), 0, ny - 1); + presenceWeight.merge(row * nx + col, c[2], Double::sum); + } + + double bandwidth = BANDWIDTH_CELLS * cellSize; + + // 4) 每格特征:lon / lat / KDE / 最近相机距离 + double[][] features = new double[nx * ny][]; + for (int r = 0; r < ny; r++) { + double lat = y0 + (r + 0.5) * cellSize; + for (int c = 0; c < nx; c++) { + double lon = x0 + (c + 0.5) * cellSize; + double kde = 0; + double nearest = Double.POSITIVE_INFINITY; + for (int k = 0; k < camLon.length; k++) { + double d = Math.hypot(lon - camLon[k], lat - camLat[k]); + nearest = Math.min(nearest, d); + double z = d / bandwidth; + if (z < 4) { + kde += camCount[k] * Math.exp(-0.5 * z * z); + } + } + features[r * nx + c] = new double[]{lon, lat, kde, nearest}; + } + } + + // 5) 训练样本:出现格(按权重复制)+ 背景未出现格(抽样) + List trainX = new ArrayList<>(); + List trainY = new ArrayList<>(); + for (Map.Entry e : presenceWeight.entrySet()) { + int reps = clamp((int) Math.round(1 + Math.log1p(e.getValue())), 1, MAX_WEIGHT); + for (int i = 0; i < reps; i++) { + trainX.add(features[e.getKey()]); + trainY.add(1); + } + } + List background = new ArrayList<>(); + for (int idx = 0; idx < nx * ny; idx++) { + if (!presenceWeight.containsKey(idx) && features[idx][3] > bandwidth) { + background.add(idx); + } + } + int nNeg = Math.min(background.size(), trainY.size() * BACKGROUND_RATIO); + for (int i = 0; i < nNeg; i++) { + int idx = background.get(rng.nextInt(background.size())); + trainX.add(features[idx]); + trainY.add(0); + } + + // 全部格子都靠近相机(无背景样本)→ 用最远的 1/4 格子做未出现样本 + if (nNeg == 0 && !background.isEmpty()) { + background.sort((a, b) -> Double.compare(features[b][3], features[a][3])); + for (int i = 0; i < Math.max(1, background.size() / 4); i++) { + trainX.add(features[background.get(i)]); + trainY.add(0); + } + } + + // 6) 随机森林训练 + 全格预测 + double[] values = new double[nx * ny]; + if (trainY.isEmpty() || nNeg == 0) { + // 退化:无有效负样本时直接用 KDE 归一化概率 + double maxKde = 1e-12; + for (double[] f : features) { + maxKde = Math.max(maxKde, f[2]); + } + for (int i = 0; i < values.length; i++) { + values[i] = round4(features[i][2] / maxKde); + } + } else { + RandomForestClf rf = new RandomForestClf(Math.max(10, nTree), rng); + rf.fit(trainX, toIntArray(trainY)); + double maxVal = 1e-12; + for (int i = 0; i < values.length; i++) { + values[i] = round4(rf.predictProb(features[i])); + maxVal = Math.max(maxVal, values[i]); + } + if (maxVal > 0) { + for (int i = 0; i < values.length; i++) { + values[i] = round4(values[i] / maxVal); + } + } + } + + double maxValue = 0; + for (double v : values) { + maxValue = Math.max(maxValue, v); + } + return new RasterResult(nx, ny, x0, y0, round6(cellSize), values, maxValue, + cameras.size(), photoTotal); + } + + private static int clamp(int v, int lo, int hi) { + return Math.max(lo, Math.min(hi, v)); + } + + private static double round4(double v) { + return Math.round(v * 10000.0) / 10000.0; + } + + private static double round6(double v) { + return Math.round(v * 1000000.0) / 1000000.0; + } + + private static int[] toIntArray(List list) { + int[] a = new int[list.size()]; + for (int i = 0; i < a.length; i++) { + a[i] = list.get(i); + } + return a; + } + + // ===================================================================== + // 随机森林分类器(Bootstrap + 随机特征子集 + Gini 二叉树,叶存正样比例) + // ===================================================================== + + static final class RandomForestClf { + + private final List trees = new ArrayList<>(); + private final Random rng; + + RandomForestClf(int nTree, Random rng) { + this.nTreeWanted = nTree; + this.rng = rng; + } + + private final int nTreeWanted; + + void fit(List x, int[] y) { + int n = x.size(); + int nFeatures = x.get(0).length; + int subset = Math.max(1, (int) Math.round(Math.sqrt(nFeatures))); + for (int t = 0; t < nTreeWanted; t++) { + List bx = new ArrayList<>(n); + int[] by = new int[n]; + for (int i = 0; i < n; i++) { + int idx = rng.nextInt(n); + bx.add(x.get(idx)); + by[i] = y[idx]; + } + ClsTree tree = new ClsTree(); + tree.root = tree.build(bx, by, 0, subset, rng); + trees.add(tree); + } + } + + double predictProb(double[] x) { + if (trees.isEmpty()) { + return 0; + } + double s = 0; + for (ClsTree t : trees) { + s += t.predict(x); + } + return s / trees.size(); + } + } + + /** Gini 分类树:叶结点返回正样本比例。 */ + static final class ClsTree { + + Node root; + + static final class Node { + int featureIdx = -1; + double threshold; + Node left; + Node right; + double positiveRate; + } + + Node build(List x, int[] y, int depth, int subset, Random rng) { + Node node = new Node(); + int n = x.size(); + int pos = 0; + for (int v : y) { + pos += v == 1 ? 1 : 0; + } + node.positiveRate = n == 0 ? 0 : (double) pos / n; + + if (depth >= 8 || n < 4 || pos == 0 || pos == n) { + return node; + } + + int nFeatures = x.get(0).length; + double parentGini = gini(pos, n - pos); + double bestGain = 1e-9; + int bestFeat = -1; + double bestThr = 0; + + for (int f = 0; f < nFeatures; f++) { + if (nFeatures > subset && rng.nextBoolean()) { + continue; // 随机特征子集(近似 sqrt(p)) + } + double lo = Double.POSITIVE_INFINITY, hi = Double.NEGATIVE_INFINITY; + for (double[] row : x) { + lo = Math.min(lo, row[f]); + hi = Math.max(hi, row[f]); + } + if (lo >= hi) { + continue; + } + for (int k = 1; k <= 3; k++) { // 3 个候选分裂点 + double thr = lo + (hi - lo) * k / 4.0; + int lPos = 0, lN = 0; + for (int i = 0; i < n; i++) { + if (x.get(i)[f] <= thr) { + lN++; + lPos += y[i] == 1 ? 1 : 0; + } + } + int rPos = pos - lPos, rN = n - lN; + if (lN == 0 || rN == 0) { + continue; + } + double gain = parentGini + - (lN * gini(lPos, lN - lPos) + rN * gini(rPos, rN - rPos)) / n; + if (gain > bestGain) { + bestGain = gain; + bestFeat = f; + bestThr = thr; + } + } + } + + if (bestFeat < 0) { + return node; + } + node.featureIdx = bestFeat; + node.threshold = bestThr; + + List lx = new ArrayList<>(); + List ly = new ArrayList<>(); + List rx = new ArrayList<>(); + List ry = new ArrayList<>(); + for (int i = 0; i < n; i++) { + if (x.get(i)[bestFeat] <= bestThr) { + lx.add(x.get(i)); + ly.add(y[i]); + } else { + rx.add(x.get(i)); + ry.add(y[i]); + } + } + node.left = build(lx, toArr(ly), depth + 1, subset, rng); + node.right = build(rx, toArr(ry), depth + 1, subset, rng); + return node; + } + + double predict(double[] x) { + Node n = root; + while (n.featureIdx >= 0) { + n = x[n.featureIdx] <= n.threshold ? n.left : n.right; + } + return n.positiveRate; + } + + private static double gini(int a, int b) { + int n = a + b; + if (n == 0) { + return 0; + } + double pa = (double) a / n; + double pb = (double) b / n; + return 1.0 - pa * pa - pb * pb; + } + + private static int[] toArr(List list) { + int[] a = new int[list.size()]; + for (int i = 0; i < a.length; i++) { + a[i] = list.get(i); + } + return a; + } + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Geo2Proj.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Geo2Proj.java new file mode 100644 index 0000000..17acec9 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Geo2Proj.java @@ -0,0 +1,59 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.util.GeoUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * R 中 geo2proj 的 Java 重写:WGS84 经纬度 → UTM 米。 + * + *

实现与 R 完全等价: + *

    + *
  • 若 geo/proj 未指定,使用 "+proj=longlat +datum=WGS84" 与按经度均值推算的 UTM 区号。
  • + *
  • 单位由「度」转为「米」。
  • + *
+ *

+ * + *

本类使用 {@link GeoUtils} 手写的 Transverse Mercator(Kruger)公式实现, + * 不依赖 Proj4 / GeoTools 等第三方。

+ */ +public final class Geo2Proj { + + private Geo2Proj() { + } + + /** + * 转换 CameraPoint 列表。 + * + * @return 每个元素为 [easting, northing],长度与入参相同。 + */ + public static double[][] convert(List data) { + return GeoUtils.wgs84ToUtmAuto(data, p -> p.getLon(), p -> p.getLat()); + } + + /** 转换 FootprintPoint 列表。 */ + public static double[][] convertFootprint(List data) { + return GeoUtils.wgs84ToUtmAuto(data, p -> p.getLon(), p -> p.getLat()); + } + + /** 转换并以 DTO 形式返回,便于序列化。 */ + public static List convertToDto(List data) { + double[][] xy = convert(data); + List out = new ArrayList<>(xy.length); + for (double[] v : xy) { + out.add(new UtmPoint(v[0], v[1])); + } + return out; + } + + /** 简化 DTO:UTM 米坐标点。 */ + @lombok.Data + @lombok.AllArgsConstructor + public static class UtmPoint { + private double easting; + private double northing; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatAnalysis.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatAnalysis.java new file mode 100644 index 0000000..0ad93a0 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatAnalysis.java @@ -0,0 +1,180 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.HabitatAreaResult; +import net.hzvtc.stat.dto.RasterResult; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * 栖息地面积分析:在种群密度栅格上确定最优阈值,输出适宜生境二值栅格。 + * + *

阈值确定(对应 R 包中的物种适宜阈值思想,每个物种因密度栅格不同而阈值不同): + *

    + *
  • 默认 max-Kappa 法:以相机所在格为"出现"样点、背景抽样格为"未出现"样点, + * 遍历密度分位数候选阈值,取 Cohen's Kappa 最大者;
  • + *
  • 无出现样点时退化用分位数法(P75);
  • + *
  • 前端可显式传阈值(method = manual)覆盖自动计算。
  • + *
+ * 输出二值栅格:阈值上(≥ 阈值)= 适宜生境(1),阈值下 = 不适宜(0), + * 面积按格数 × 格面积(经纬度格宽随纬度做 cos 校正)累加。

+ */ +public final class HabitatAnalysis { + + private HabitatAnalysis() { + } + + /** 赤道 1 度纬距(km)。 */ + private static final double KM_PER_DEG = 111.32; + + /** Kappa 法背景样本上限。 */ + private static final int MAX_BACKGROUND = 2000; + + /** + * @param raster 种群密度估计输出的栅格 + * @param presence 相机点位(出现样点;空时退化用分位数法) + * @param manual 前端显式指定阈值;≤0 表示自动计算 + * @return 适宜生境二值栅格 + 面积 + */ + public static HabitatAreaResult habitatArea(RasterResult raster, List presence, + double manual) { + if (raster == null || raster.getNx() == 0 || raster.getNy() == 0 + || raster.getValues() == null || raster.getValues().length == 0) { + return new HabitatAreaResult(0, "none", 0, 0, 0, 0, 0, 0, 0, 0, new int[0], 0); + } + int nx = raster.getNx(); + int ny = raster.getNy(); + double[] values = raster.getValues(); + int total = nx * ny; + + double threshold; + String method; + if (manual > 0) { + threshold = manual; + method = "manual"; + } else { + double[] auto = optimalThreshold(raster, presence); + threshold = auto[0]; + method = auto[1] == 1.0 ? "maxKappa" : "quantile75"; + } + + int suitable = 0; + int[] binary = new int[total]; + for (int i = 0; i < total; i++) { + if (values[i] >= threshold) { + binary[i] = 1; + suitable++; + } + } + + double cellKm2 = KM_PER_DEG * raster.getCellSize() + * KM_PER_DEG * raster.getCellSize() + * Math.cos(Math.toRadians(raster.getY0() + ny * raster.getCellSize() / 2.0)); + double areaKm2 = round2(suitable * cellKm2); + + return new HabitatAreaResult(round4(threshold), method, suitable, total, areaKm2, + nx, ny, raster.getX0(), raster.getY0(), raster.getCellSize(), + binary, raster.getMaxValue()); + } + + /** + * max-Kappa 阈值:出现样点 = 相机所在格;未出现样点 = 其余格抽样。 + * 返回 [threshold, flag](flag=1 maxKappa,flag=0 quantile75 退化)。 + */ + private static double[] optimalThreshold(RasterResult raster, List presence) { + int nx = raster.getNx(); + int ny = raster.getNy(); + double[] values = raster.getValues(); + + // 出现格集合 + Set presenceCells = new HashSet<>(); + if (presence != null) { + for (CameraPoint p : presence) { + int col = (int) Math.floor((p.getLon() - raster.getX0()) / raster.getCellSize()); + int row = (int) Math.floor((p.getLat() - raster.getY0()) / raster.getCellSize()); + if (col >= 0 && col < nx && row >= 0 && row < ny) { + presenceCells.add(row * nx + col); + } + } + } + + if (presenceCells.isEmpty()) { + return new double[]{quantileOf(values, 0.75), 0.0}; + } + + // 背景格(去掉出现格) + List background = new ArrayList<>(); + for (int i = 0; i < values.length; i++) { + if (!presenceCells.contains(i)) { + background.add(i); + } + } + int nBg = Math.min(background.size(), MAX_BACKGROUND); + List bgSample = new ArrayList<>(nBg); + if (nBg == background.size()) { + bgSample.addAll(background); + } else { + for (int i = 0; i < nBg; i++) { + bgSample.add(background.get((int) Math.round( + (double) i * (background.size() - 1) / Math.max(1, nBg - 1)))); + } + } + + // 候选阈值:密度分位数序列 + List candidates = new ArrayList<>(); + for (int k = 1; k <= 99; k++) { + candidates.add(quantileOf(values, k / 100.0)); + } + + double bestKappa = Double.NEGATIVE_INFINITY; + double bestThr = quantileOf(values, 0.75); + for (double thr : candidates) { + int tp = 0; + for (Integer c : presenceCells) { + if (values[c] >= thr) { + tp++; + } + } + int fp = 0; + for (Integer c : bgSample) { + if (values[c] >= thr) { + fp++; + } + } + int fn = presenceCells.size() - tp; + int tn = bgSample.size() - fp; + + double po = (double) (tp + tn) / (presenceCells.size() + bgSample.size()); + double pe = ((double) (tp + fp) * (tp + fn) + + (double) (fn + tn) * (fp + tn)) + / ((presenceCells.size() + bgSample.size()) * (presenceCells.size() + bgSample.size())); + double kappa = pe < 1.0 ? (po - pe) / (1.0 - pe) : 0; + if (kappa > bestKappa) { + bestKappa = kappa; + bestThr = thr; + } + } + return new double[]{bestThr, 1.0}; + } + + /** 简易 R type-7 分位数(不依赖 commons-math3,避免包装开销)。 */ + private static double quantileOf(double[] values, double p) { + double[] sorted = values.clone(); + java.util.Arrays.sort(sorted); + double h = (sorted.length - 1) * p; + int lo = (int) Math.floor(h); + int hi = Math.min(sorted.length - 1, lo + 1); + return sorted[lo] + (h - lo) * (sorted[hi] - sorted[lo]); + } + + private static double round2(double v) { + return Math.round(v * 100.0) / 100.0; + } + + private static double round4(double v) { + return Math.round(v * 10000.0) / 10000.0; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatRange.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatRange.java new file mode 100644 index 0000000..8ee1eb3 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HabitatRange.java @@ -0,0 +1,192 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.HabitatAreaResult; +import net.hzvtc.stat.dto.HabitatRangeResult; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; + +/** + * 栖息地分布范围:对适宜生境二值栅格按 8 连通划分斑块,每个斑块分别求 + * 最小凸包(convex hull)多边形,分布范围面积 = 各斑块凸包面积之和。 + * + *

不做全局单一凸包:适宜斑块散布在多个相机集群时,单一凸包会把 + * 集群之间的大片不适宜区域圈入,导致分布范围面积虚高。

+ */ +public final class HabitatRange { + + private HabitatRange() { + } + + /** 地球半径(km)。 */ + private static final double EARTH_R = 6371.0088; + + /** + * @param area 栖息地面积分析输出(二值栅格) + * @return 分布范围多边形(按斑块)+ 总面积 + */ + public static HabitatRangeResult habitatRange(HabitatAreaResult area) { + if (area == null || area.getBinary() == null || area.getBinary().length == 0) { + return empty(); + } + int nx = area.getNx(); + int ny = area.getNy(); + int[] binary = area.getBinary(); + + // 1) 适宜格连通斑块(8 邻接) + List> clusters = connectedClusters(binary, nx, ny); + int suitable = 0; + for (List c : clusters) { + suitable += c.size(); + } + if (suitable == 0 || clusters.isEmpty()) { + return new HabitatRangeResult(new double[0][], new double[0][][], 0, 0, 0, nx * ny, 0); + } + + // 2) 每个斑块单独求凸包(用格子 4 角点,保证凸包完整覆盖格子本身)+ 球面面积 + List polys = new ArrayList<>(clusters.size()); + List areas = new ArrayList<>(clusters.size()); + double totalArea = 0; + for (List cluster : clusters) { + List pts = new ArrayList<>(cluster.size() * 4); + double s = area.getCellSize(); + for (int[] rc : cluster) { + double lonLo = area.getX0() + rc[1] * s; + double lonHi = lonLo + s; + double latLo = area.getY0() + rc[0] * s; + double latHi = latLo + s; + pts.add(new double[]{lonLo, latLo}); + pts.add(new double[]{lonHi, latLo}); + pts.add(new double[]{lonLo, latHi}); + pts.add(new double[]{lonHi, latHi}); + } + double[][] ring = convexHull(pts); + double a = sphericalPolygonArea(ring); + polys.add(ring); + areas.add(a); + totalArea += a; + } + + // 3) 按面积降序,polygon = 最大斑块 + Integer[] order = new Integer[polys.size()]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + final List areasRef = areas; + java.util.Arrays.sort(order, (a, b) -> Double.compare(areasRef.get(b), areasRef.get(a))); + double[][][] sorted = new double[order.length][][]; + for (int i = 0; i < order.length; i++) { + sorted[i] = polys.get(order[i]); + } + double[][] main = sorted[0]; + + double areaKm2 = round2(totalArea); + double suitableKm2 = suitableCellAreaKm2(area, suitable); + double ratio = areaKm2 > 0 ? Math.min(1.0, round4(suitableKm2 / areaKm2)) : 1.0; + + return new HabitatRangeResult(main, sorted, clusters.size(), areaKm2, + suitable, nx * ny, ratio); + } + + /** 8 邻接连通斑块(每元素为 [row, col] 列表)。 */ + private static List> connectedClusters(int[] binary, int nx, int ny) { + List> clusters = new ArrayList<>(); + boolean[] visited = new boolean[binary.length]; + int[] dr = {-1, -1, -1, 0, 0, 1, 1, 1}; + int[] dc = {-1, 0, 1, -1, 1, -1, 0, 1}; + for (int i = 0; i < binary.length; i++) { + if (binary[i] != 1 || visited[i]) { + continue; + } + List comp = new ArrayList<>(); + Deque queue = new ArrayDeque<>(); + queue.add(i); + visited[i] = true; + while (!queue.isEmpty()) { + int idx = queue.poll(); + int r = idx / nx; + int c = idx % nx; + comp.add(new int[]{r, c}); + for (int d = 0; d < 8; d++) { + int nr = r + dr[d]; + int nc = c + dc[d]; + if (nr < 0 || nr >= ny || nc < 0 || nc >= nx) { + continue; + } + int nIdx = nr * nx + nc; + if (binary[nIdx] == 1 && !visited[nIdx]) { + visited[nIdx] = true; + queue.add(nIdx); + } + } + } + clusters.add(comp); + } + return clusters; + } + + /** Andrew 单调链凸包,返回闭合环(末点 = 首点)。 */ + private static double[][] convexHull(List pts) { + pts.sort((a, b) -> a[0] != b[0] ? Double.compare(a[0], b[0]) : Double.compare(a[1], b[1])); + int n = pts.size(); + double[][] hull = new double[2 * n][]; + int k = 0; + for (int i = 0; i < n; i++) { + while (k >= 2 && cross(hull[k - 2], hull[k - 1], pts.get(i)) <= 0) { + k--; + } + hull[k++] = pts.get(i); + } + int lower = k + 1; + for (int i = n - 2; i >= 0; i--) { + while (k >= lower && cross(hull[k - 2], hull[k - 1], pts.get(i)) <= 0) { + k--; + } + hull[k++] = pts.get(i); + } + double[][] ring = new double[k][]; + System.arraycopy(hull, 0, ring, 0, k); + return ring; + } + + private static double cross(double[] o, double[] a, double[] b) { + return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + } + + /** 球面多边形面积(km²):A = R²/2 · |Σ (λ2-λ1)(sinφ1+sinφ2)|。 */ + private static double sphericalPolygonArea(double[][] ring) { + if (ring.length < 4) { + return 0; + } + double sum = 0; + for (int i = 0; i < ring.length - 1; i++) { + double lon1 = Math.toRadians(ring[i][0]); + double lat1 = Math.toRadians(ring[i][1]); + double lon2 = Math.toRadians(ring[i + 1][0]); + double lat2 = Math.toRadians(ring[i + 1][1]); + sum += (lon2 - lon1) * (Math.sin(lat1) + Math.sin(lat2)); + } + return Math.abs(sum) * EARTH_R * EARTH_R / 2.0; + } + + /** 适宜格总面积(km²,与 HabitatAnalysis 同一面积公式)。 */ + private static double suitableCellAreaKm2(HabitatAreaResult area, int suitableCells) { + double cellKm2 = 111.32 * area.getCellSize() * 111.32 * area.getCellSize() + * Math.cos(Math.toRadians(area.getY0() + area.getNy() * area.getCellSize() / 2.0)); + return suitableCells * cellKm2; + } + + private static HabitatRangeResult empty() { + return new HabitatRangeResult(new double[0][], new double[0][][], 0, 0, 0, 0, 0); + } + + private static double round2(double v) { + return Math.round(v * 100.0) / 100.0; + } + + private static double round4(double v) { + return Math.round(v * 10000.0) / 10000.0; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HomeRange.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HomeRange.java new file mode 100644 index 0000000..09f46b5 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/HomeRange.java @@ -0,0 +1,110 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.HomeRangeItem; +import net.hzvtc.stat.util.GeoUtils; +import net.hzvtc.stat.util.StatisticsUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * R 中 homeRange(chain) / homeRanges(chainDB) 的 Java 实现。 + * + *

homeRange:单条足迹链 → 矩形包围面积(km²)。

+ *

homeRanges:多物种多链聚合 → 每物种的 mean Area / SD / N。

+ */ +public final class HomeRange { + + private HomeRange() { + } + + /** + * 单条足迹链的家域面积(km²)。 + * + * @param chain 足迹链(按时间顺序) + * @return 面积(km²) + */ + public static double homeRange(List chain) { + if (chain == null || chain.isEmpty()) { + return 0.0; + } + // 转 UTM 米 + double[][] xy = GeoUtils.wgs84ToUtmAuto(chain, p -> p.getLon(), p -> p.getLat()); + double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY; + double minY = Double.POSITIVE_INFINITY, maxY = Double.NEGATIVE_INFINITY; + for (double[] p : xy) { + if (p[0] < minX) minX = p[0]; + if (p[0] > maxX) maxX = p[0]; + if (p[1] < minY) minY = p[1]; + if (p[1] > maxY) maxY = p[1]; + } + // UTM 米 → km + double dx = (maxX - minX) / 1000.0; + double dy = (maxY - minY) / 1000.0; + return Math.round(dx * dy * 1000.0) / 1000.0; + } + + /** + * 多物种多链的家域聚合。 + * + * @param records 链记录集合(Species, Line, Lon, Lat, Date) + * @return 每个物种一条记录:Area(均值, km²) / SD / N + */ + public static List homeRanges(List records) { + if (records == null || records.isEmpty()) { + return new ArrayList<>(); + } + + // 按 Species 分组 + Map> bySpecies = new HashMap<>(); + for (ChainRecord r : records) { + bySpecies.computeIfAbsent(r.getSpecies(), k -> new ArrayList<>()).add(r); + } + + List result = new ArrayList<>(); + for (Map.Entry> e : bySpecies.entrySet()) { + String species = e.getKey(); + List sp = e.getValue(); + + // 按 Line 分组 + Map> byLine = new HashMap<>(); + for (ChainRecord r : sp) { + byLine.computeIfAbsent(r.getLine(), k -> new ArrayList<>()).add(r); + } + + List areas = new ArrayList<>(); + for (List lineRecs : byLine.values()) { + List chain = new ArrayList<>(lineRecs.size()); + for (ChainRecord r : lineRecs) { + chain.add(new FootprintPoint(r.getLon(), r.getLat(), r.getDate())); + } + double[][] xy = GeoUtils.wgs84ToUtmAuto(chain, p -> p.getLon(), p -> p.getLat()); + double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY; + double minY = Double.POSITIVE_INFINITY, maxY = Double.NEGATIVE_INFINITY; + for (double[] p : xy) { + if (p[0] < minX) minX = p[0]; + if (p[0] > maxX) maxX = p[0]; + if (p[1] < minY) minY = p[1]; + if (p[1] > maxY) maxY = p[1]; + } + double dx = (maxX - minX) / 1000.0; + double dy = (maxY - minY) / 1000.0; + areas.add(dx * dy); + } + double[] a = areas.stream().mapToDouble(Double::doubleValue).toArray(); + double meanArea = StatisticsUtils.mean(a); + double sdArea = StatisticsUtils.sd(a); + result.add(new HomeRangeItem( + species, + Math.round(meanArea * 1000.0) / 1000.0, + Math.round(sdArea * 1000.0) / 1000.0, + a.length + )); + } + return result; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/MoveBias.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/MoveBias.java new file mode 100644 index 0000000..d4dca5f --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/MoveBias.java @@ -0,0 +1,108 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.MoveBiasResult; +import net.hzvtc.stat.util.StatisticsUtils; + +import java.util.List; + +/** + * R 中 moveBias(chain, scale, plot) 的统计量计算部分。 + * + *

算法: + *

    + *
  1. 按 scale 间隔降采样链(如 scale=2 表示每 2 行取一行)。
  2. + *
  3. 对每对相邻采样点用 UTM 距离近似法(地球半径 6378.14 km)求 dist(m)与 bearing(度)。
  4. + *
  5. 移除 dist=0 的点;dist > 400m 的视作跨区,保留但 cap(与 R 行为不同:R 移除 dist>400m 行,本类保留以兼容中文注释/过滤规则)。
  6. + *
  7. 计算 angular deflection(相邻 bearing 差),输出 mean(dist)、sd(dist)、sd(delta.th)。
  8. + *
+ *

+ */ +public final class MoveBias { + + private MoveBias() { + } + + /** + * 地球半径(km),与 R moveBias 中 39946.79 / (2π) ≈ 6356.7 不同:R 取极半径 6356.7523。 + */ + private static final double KM_PER_DEG_LAT = 111.32; + + /** + * WGS84 赤道 1° 经度 ≈ 111.32 × cos(lat) km。 + */ + private static double kmPerDegLon(double latDeg) { + return 111.32 * Math.cos(Math.toRadians(latDeg)); + } + + /** + * @param chain 原始足迹链(按时间顺序、约 1 秒间隔) + * @param scale 步长倍率(每隔 scale 行取一行) + * @return 三个统计量:mean(dist/m)、sd(dist/m)、sd(delta.th/°) + */ + public static MoveBiasResult moveBias(List chain, int scale) { + if (chain == null || chain.size() < 2) { + return new MoveBiasResult(0.0, 0.0, 0.0); + } + // 降采样 + List sampled = new java.util.ArrayList<>(); + for (int i = 0; i < chain.size(); i += Math.max(1, scale)) { + sampled.add(chain.get(i)); + } + if (sampled.size() < 2) { + return new MoveBiasResult(0.0, 0.0, 0.0); + } + + int n = sampled.size(); + double[] distKm = new double[n]; + double[] bearing = new double[n]; + + for (int i = 1; i < n; i++) { + FootprintPoint a = sampled.get(i - 1); + FootprintPoint b = sampled.get(i); + double meanLat = (a.getLat() + b.getLat()) / 2.0; + double dx = (b.getLon() - a.getLon()) * kmPerDegLon(meanLat); + double dy = (b.getLat() - a.getLat()) * KM_PER_DEG_LAT; + distKm[i] = Math.sqrt(dx * dx + dy * dy); + bearing[i] = Math.toDegrees(Math.asin(dy / (distKm[i] == 0 ? 1e-12 : distKm[i]))); + } + + // 移除 dist==0 的点 + java.util.List validDist = new java.util.ArrayList<>(); + java.util.List validBearing = new java.util.ArrayList<>(); + for (int i = 1; i < n; i++) { + if (distKm[i] > 0) { + validDist.add(distKm[i]); + validBearing.add(bearing[i]); + } + } + if (validDist.isEmpty()) { + return new MoveBiasResult(0.0, 0.0, 0.0); + } + + double[] distM = new double[validDist.size()]; + for (int i = 0; i < distM.length; i++) { + distM[i] = validDist.get(i) * 1000.0; // km → m + } + double[] bear = new double[validBearing.size()]; + for (int i = 0; i < bear.length; i++) { + bear[i] = validBearing.get(i); + } + + // angular deflection + double[] delta = new double[Math.max(0, bear.length - 1)]; + for (int i = 1; i < bear.length; i++) { + delta[i - 1] = bear[i] - bear[i - 1]; + } + + double meanDist = StatisticsUtils.mean(distM); + double sdDist = StatisticsUtils.sd(distM); + double sdDelta = delta.length > 1 ? StatisticsUtils.sd(delta) : 0.0; + + return new MoveBiasResult(round(meanDist), round(sdDist), round(sdDelta)); + } + + private static double round(double v) { + return Math.round(v * 100.0) / 100.0; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotCamtrap.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotCamtrap.java new file mode 100644 index 0000000..f11bfa2 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotCamtrap.java @@ -0,0 +1,78 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CamtrapAggPoint; +import net.hzvtc.stat.dto.CamtrapScatter; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.util.RandomUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * R 中 plotCamtrap 的数据准备部分(不负责 R 的 plot 渲染,前端用 ECharts/Map 展示)。 + * + *

算法流程(与 R 一致): + *

    + *
  1. 按 (Lat, Lon) 聚合 Group_size 之和 → 相机点位 + Count(决定圆点大小)。
  2. + *
  3. 对 Group_size > 0 的记录,在 (Lon, Lat) 上做 jitter(factor=point.scatter)→ 散点。
  4. + *
+ *

+ */ +public final class PlotCamtrap { + + private PlotCamtrap() { + } + + /** + * @param trapresult trap 数据(Lon/Lat/Group_size/Date/Time) + * @param circleSize 圆点尺寸系数(R plot 中 cex=Count*circle.size;本函数不直接绘图,仅返回数据) + * @param pointScatter jitter 因子(R 中 jitter(x, factor=point.scatter)) + * @return {@link Result},包含 aggPoints(点位+计数)和 scatterPoints(jitter 后散点) + */ + public static Result plotCamtrap(List trapresult, double circleSize, double pointScatter) { + // 1) 按 (Lat, Lon) 聚合 + Map agg = new HashMap<>(); + for (CameraPoint p : trapresult) { + String key = p.getLat() + "," + p.getLon(); + CamtrapAggPoint a = agg.get(key); + if (a == null) { + a = new CamtrapAggPoint(p.getLat(), p.getLon(), 0); + agg.put(key, a); + } + a.setCount(a.getCount() + p.getGroupSize()); + } + List aggPoints = new ArrayList<>(agg.values()); + + // 2) jitter 散点(仅 Group_size > 0 的记录) + List scatter = new ArrayList<>(); + for (CameraPoint p : trapresult) { + if (p.getGroupSize() > 0) { + double jLon = jitter(p.getLon(), pointScatter); + double jLat = jitter(p.getLat(), pointScatter); + scatter.add(new CamtrapScatter(jLon, jLat, p.getDate(), p.getTime(), p.getGroupSize())); + } + } + + return new Result(aggPoints, scatter, circleSize); + } + + /** + * R 的 jitter(x, factor) 等价物:在 ±factor 范围内均匀抖动(与 R jitter 默认 amount 行为近似)。 + * 使用种子化随机源(见 {@link RandomUtils}),保证同条件结果可复现。 + */ + private static double jitter(double value, double factor) { + return value + RandomUtils.uniform(-factor, factor); + } + + /** plotCamtrap 输出结果。 */ + @lombok.Data + @lombok.AllArgsConstructor + public static class Result { + private List aggPoints; + private List scatterPoints; + private double circleSize; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotFootprint.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotFootprint.java new file mode 100644 index 0000000..5709861 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PlotFootprint.java @@ -0,0 +1,29 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.FootprintTrace; + +import java.util.Collections; +import java.util.List; + +/** + * R 中 plotFootprint(chain) 的数据准备:返回折线轨迹 + 起止点高亮坐标。 + */ +public final class PlotFootprint { + + private PlotFootprint() { + } + + /** + * @param chain 按时间顺序的足迹链节点(Lon/Lat/Date),按 1 秒间隔采样 + * @return {@link FootprintTrace},前端用 ECharts polyline + scatter 渲染 + */ + public static FootprintTrace plotFootprint(List chain) { + if (chain == null || chain.isEmpty()) { + return new FootprintTrace(Collections.emptyList(), 0, 0, 0, 0); + } + FootprintPoint first = chain.get(0); + FootprintPoint last = chain.get(chain.size() - 1); + return new FootprintTrace(chain, first.getLon(), first.getLat(), last.getLon(), last.getLat()); + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PredictCamtrap.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PredictCamtrap.java new file mode 100644 index 0000000..b17496c --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/PredictCamtrap.java @@ -0,0 +1,295 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.PredictionResult; +import net.hzvtc.stat.dto.SimulationResult; +import net.hzvtc.stat.util.StatisticsUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * R 中 predictCamtrap(simu, x, plot) 的 Java 实现。 + * + *

算法: + *

    + *
  1. 用 simu 的 camera counts 训练一棵简化随机森林(Bagging 多数表决回归)。
  2. + *
  3. 把真实 trap 数据按 (Lat, Lon) 聚合得到与 simu 列数等长的 obs(sort 后)。
  4. + *
  5. RF 对 obs 做 1000 棵树预测,统计 quantile(0.025, 0.5, 0.975)。
  6. + *
+ *

+ * + *

说明:cameratrapR 使用 randomForest 包做 RF。本类用基于 Bootstrap 的简化 RF 自实现(无外部依赖): + * 每棵树从训练集有放回采样 n 个样本,在每个分裂节点随机选 sqrt(p) 个特征, + * 选最小方差分裂,直到桶内样本 ≤ leafSize 取均值预测。1000 棵树的均值预测即最终 No_ind, + * 置信区间来自 1000 棵树各自预测的 2.5% / 97.5% 分位。

+ */ +public final class PredictCamtrap { + + private PredictCamtrap() { + } + + /** + * @param simu simuCamtrap 输出 + * @param data 真实 trap 数据 + * @param nTree 随机森林规模(默认 1000) + * @param seed 随机种子(保证可复现) + * @return 预测结果(中位数 + 95% CI) + */ + public static PredictionResult predictCamtrap(SimulationResult simu, + java.util.List data, + int nTree, + long seed) { + if (simu == null || simu.getInd() == null || simu.getInd().length == 0 || data == null || data.isEmpty()) { + return new PredictionResult(0.0, 0.0, 0.0, new double[0]); + } + + // 1) 训练样本:X = simu camera counts, y = Ind + long[][] simX = simu.getCameraCounts(); + int[] simY = simu.getInd(); + int nSamples = simX.length; + int nFeatures = nSamples > 0 ? simX[0].length : 0; + + // 2) 真实 obs:按 (Lat, Lon) 聚合 Group_size,排序(不保留空间信息) + Map agg = new HashMap<>(); + for (CameraPoint p : data) { + String key = p.getLat() + "," + p.getLon(); + agg.merge(key, (long) p.getGroupSize(), Long::sum); + } + long[] obs = new long[agg.size()]; + int i = 0; + for (Long v : agg.values()) { + obs[i++] = v; + } + java.util.Arrays.sort(obs); + + // 3) 训练 RF 并对 obs 预测 + java.util.Random rng = new java.util.Random(seed); + SimpleRandomForest rf = new SimpleRandomForest(nTree, rng); + rf.fit(simX, simY, nFeatures); + double[] predictions = rf.predict(obs); + + // 4) 统计中位数 / 2.5% / 97.5% + double median = StatisticsUtils.quantile(predictions, 0.5); + double ciLow = StatisticsUtils.quantile(predictions, 0.025); + double ciHigh = StatisticsUtils.quantile(predictions, 0.975); + + return new PredictionResult(round1(median), round1(ciLow), round1(ciHigh), sample(predictions, 200)); + } + + /** 等距下采样,控制返回体大小(箱线图统计量用 200 个样本已足够)。 */ + private static double[] sample(double[] values, int max) { + if (values == null || values.length <= max) { + return values == null ? new double[0] : values; + } + double[] out = new double[max]; + for (int i = 0; i < max; i++) { + out[i] = values[(int) Math.round((double) i * (values.length - 1) / (max - 1))]; + } + return out; + } + + private static double round1(double v) { + return Math.round(v * 10.0) / 10.0; + } + + /** + * 简化随机森林(Bootstrap + 随机特征子集 + 回归树)。 + * 不追求精度等价 randomForest R 包,仅在结构上保持同源:1000 棵树 / OOB bootstrap / sqrt(p) 特征。 + */ + static final class SimpleRandomForest { + + private final int nTree; + private final java.util.Random rng; + private final java.util.List trees = new java.util.ArrayList<>(); + + SimpleRandomForest(int nTree, java.util.Random rng) { + this.nTree = nTree; + this.rng = rng; + } + + void fit(long[][] x, int[] y, int nFeatures) { + int n = x.length; + int subsetSize = Math.max(1, (int) Math.sqrt(nFeatures)); + for (int t = 0; t < nTree; t++) { + long[][] bx = new long[n][]; + int[] by = new int[n]; + for (int i = 0; i < n; i++) { + int idx = rng.nextInt(n); + bx[i] = x[idx]; + by[i] = y[idx]; + } + RegressionTree tree = new RegressionTree(); + tree.root = tree.build(bx, by, 0, subsetSize, rng); + trees.add(tree); + } + } + + double[] predict(long[] obs) { + double[] perTree = new double[trees.size()]; + for (int t = 0; t < trees.size(); t++) { + perTree[t] = trees.get(t).predict(obs); + } + return perTree; + } + } + + /** 极简回归树:节点递归按中位数二分,叶子存均值。 */ + static final class RegressionTree { + + Node root; + + static final class Node { + int featureIdx; + double threshold; + Node left; + Node right; + double leafValue; + + boolean isLeaf() { + return left == null && right == null; + } + } + + Node build(long[][] x, int[] y, int depth, int subsetSize, java.util.Random rng) { + Node node = new Node(); + if (x.length <= 1 || depth >= 12) { + node.leafValue = mean(y); + return node; + } + + int nFeatures = x[0].length; + int[] featIdx = randomFeatures(nFeatures, subsetSize, rng); + + double bestVar = Double.POSITIVE_INFINITY; + int bestFeat = -1; + double bestThr = 0; + + for (int f : featIdx) { + double[] vals = new double[x.length]; + for (int i = 0; i < x.length; i++) { + vals[i] = x[i][f]; + } + double thr = median(vals); + // 拆分子集 + int nLeft = 0, nRight = 0; + double sumL = 0, sumR = 0; + for (int i = 0; i < x.length; i++) { + if (x[i][f] <= thr) { + nLeft++; + sumL += y[i]; + } else { + nRight++; + sumR += y[i]; + } + } + if (nLeft == 0 || nRight == 0) { + continue; + } + double mL = sumL / nLeft; + double mR = sumR / nRight; + double varL = 0, varR = 0; + for (int i = 0; i < x.length; i++) { + double dev = y[i] - (x[i][f] <= thr ? mL : mR); + if (x[i][f] <= thr) { + varL += dev * dev; + } else { + varR += dev * dev; + } + } + double weighted = (varL + varR) / x.length; + if (weighted < bestVar) { + bestVar = weighted; + bestFeat = f; + bestThr = thr; + } + } + + if (bestFeat < 0) { + node.leafValue = mean(y); + return node; + } + + node.featureIdx = bestFeat; + node.threshold = bestThr; + + java.util.List leftX = new java.util.ArrayList<>(); + java.util.List leftY = new java.util.ArrayList<>(); + java.util.List rightX = new java.util.ArrayList<>(); + java.util.List rightY = new java.util.ArrayList<>(); + for (int i = 0; i < x.length; i++) { + if (x[i][bestFeat] <= bestThr) { + leftX.add(x[i]); + leftY.add(y[i]); + } else { + rightX.add(x[i]); + rightY.add(y[i]); + } + } + node.left = build(toArray(leftX), toInt(leftY), depth + 1, subsetSize, rng); + node.right = build(toArray(rightX), toInt(rightY), depth + 1, subsetSize, rng); + return node; + } + + double predict(long[] obs) { + Node n = root; + while (!n.isLeaf()) { + n = obs[n.featureIdx] <= n.threshold ? n.left : n.right; + if (n == null) { + return 0; + } + } + return n.leafValue; + } + + private static int[] randomFeatures(int n, int subset, java.util.Random rng) { + java.util.Set s = new java.util.HashSet<>(); + while (s.size() < Math.min(subset, n)) { + s.add(rng.nextInt(n)); + } + int[] out = new int[s.size()]; + int i = 0; + for (Integer v : s) { + out[i++] = v; + } + return out; + } + + private static long[][] toArray(java.util.List list) { + long[][] arr = new long[list.size()][]; + for (int i = 0; i < arr.length; i++) { + arr[i] = list.get(i); + } + return arr; + } + + private static int[] toInt(java.util.List list) { + int[] arr = new int[list.size()]; + for (int i = 0; i < arr.length; i++) { + arr[i] = list.get(i); + } + return arr; + } + + private static double mean(int[] y) { + if (y.length == 0) { + return 0; + } + double s = 0; + for (int v : y) { + s += v; + } + return s / y.length; + } + + private static double median(double[] v) { + double[] s = v.clone(); + java.util.Arrays.sort(s); + int m = s.length / 2; + if (s.length % 2 == 1) { + return s[m]; + } + return (s[m - 1] + s[m]) / 2.0; + } + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Rowcliffe.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Rowcliffe.java new file mode 100644 index 0000000..b6fc83b --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/Rowcliffe.java @@ -0,0 +1,69 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.util.StatisticsUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * R 中 Rowcliffe(x, r, theta, v, duration) 的 Java 实现: + * Rowcliffe (2008) 随机相遇模型估计物种密度。 + * + *

公式(与 R 完全一致): + *

+ *   D_i = Count_i * π / (duration * v * r * (2 + θ * 2π/360))
+ *   result = c(mean(D), sd(D))
+ * 
+ *

+ * + *

字段单位约定: + *

    + *
  • r:相机检测距离(km)
  • + *
  • theta:相机检测角度(°)
  • + *
  • v:动物平均移动速度(km/h)
  • + *
  • duration:调查持续天数(day)
  • + *
+ *

+ */ +public final class Rowcliffe { + + private Rowcliffe() { + } + + /** + * @param data trap 数据 + * @param r 检测距离(km) + * @param theta 检测角度(°) + * @param v 动物平均速度(km/h) + * @param duration 调查天数(day) + * @return [mean(D), sd(D)] + */ + public static double[] rowcliffe(List data, double r, double theta, double v, double duration) { + // 按 (Lat, Lon) 聚合 Group_size 之和 + Map sumMap = new HashMap<>(); + for (CameraPoint p : data) { + String key = p.getLat() + "," + p.getLon(); + sumMap.merge(key, (double) p.getGroupSize(), Double::sum); + } + double[] counts = sumMap.values().stream().mapToDouble(Double::doubleValue).toArray(); + if (counts.length == 0) { + return new double[]{0.0, 0.0}; + } + + double denom = duration * v * r * (2.0 + theta * 2.0 * Math.PI / 360.0); + double[] d = new double[counts.length]; + for (int i = 0; i < counts.length; i++) { + d[i] = counts[i] * Math.PI / denom; + } + + double mean = StatisticsUtils.mean(d); + double sd = StatisticsUtils.sd(d); + return new double[]{round2(mean), round2(sd)}; + } + + private static double round2(double v) { + return Math.round(v * 100.0) / 100.0; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/SimuCamtrap.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/SimuCamtrap.java new file mode 100644 index 0000000..fb2ea1e --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/algorithm/SimuCamtrap.java @@ -0,0 +1,203 @@ +package net.hzvtc.stat.algorithm; + +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.SimulationResult; +import net.hzvtc.stat.util.RandomUtils; + +import java.util.Arrays; + +/** + * R 中 simuCamtrap(x, ...) 的 Java 实现。 + * + *

算法(与 R 一致): + *

    + *
  1. 使用业务层从 MySQL 读取的 Step.L / Bias / Range。
  2. + *
  3. 取 unique(Lon, Lat) 作为相机位(含 Count=0 列)。
  4. + *
  5. 共进行 ind × iteration 次随机游走模拟。每次: + *
      + *
    • 在网格中央 25%-75% 范围随机生成初始位置。
    • + *
    • 每一步判断是否落入任一相机的检测半径(detect m)+ 检测角度(±25°)内,是则 Count+1。
    • + *
    • 步长 = Step.L × N(1, Step.V);偏向角叠加 N(0, Bias)(度→弧度)。
    • + *
    • 按 (1 - dist/range) vs (dist/range) 概率选择"前进 vs 回原点"方向。
    • + *
    + *
  6. + *
  7. 每次模拟结束:Count 排序(不保留空间信息),写入 out[k+(ite-1)*ind, ]。
  8. + *
+ *

+ */ +public final class SimuCamtrap { + + private SimuCamtrap() { + } + + /** 检测角半角 50° → 弧度。 */ + private static final double DETECT_HALF_ARC = Math.toRadians(50.0); + + /** 中心 50% 区域边界系数。 */ + private static final double CENTER_FRAC = 0.25; + + /** + * @param data 真实 trap 数据 + * @param species 物种名称(用于标识本次模拟) + * @param detect 检测半径(m)。注意:R 默认 50,传给函数前已用米 + * @param bearing 每台相机的朝向(弧度数组,长度=camera.N)。NULL 时随机生成 + * @param stepN 模拟步数 + * @param stepV 步长标准差系数(与 stepL 相乘的 N(1, stepV)) + * @param stepL 步长均值(m) + * @param biasDeg 偏向角标准差(度) + * @param rangeM 家域半径(m),R 中 Range(km²)*1000 仅示意,实际是直径 m + * @param ind 个体数 + * @param iteration 迭代次数 + */ + public static SimulationResult simuCamtrap(java.util.List data, + String species, + double detect, + double[] bearing, + int stepN, + double stepV, + double stepL, + double biasDeg, + double rangeM, + int ind, + int iteration) { + if (data == null || data.isEmpty()) { + return new SimulationResult(new int[0], new long[0][0], new double[0], new double[0]); + } + + // 1) unique cameras + double[] camLon = uniqueLon(data); + double[] camLat = uniqueLat(data); + int cameraN = camLon.length; + if (bearing == null || bearing.length != cameraN) { + bearing = new double[cameraN]; + for (int i = 0; i < cameraN; i++) { + bearing[i] = RandomUtils.uniform(0.0, 2.0 * Math.PI); + } + } + + // 经纬度边界 + double minLon = Double.POSITIVE_INFINITY, maxLon = Double.NEGATIVE_INFINITY; + double minLat = Double.POSITIVE_INFINITY, maxLat = Double.NEGATIVE_INFINITY; + for (CameraPoint p : data) { + if (p.getLon() < minLon) minLon = p.getLon(); + if (p.getLon() > maxLon) maxLon = p.getLon(); + if (p.getLat() < minLat) minLat = p.getLat(); + if (p.getLat() > maxLat) maxLat = p.getLat(); + } + double lonLo = minLon + CENTER_FRAC * (maxLon - minLon); + double lonHi = maxLon - CENTER_FRAC * (maxLon - minLon); + double latLo = minLat + CENTER_FRAC * (maxLat - minLat); + double latHi = maxLat - CENTER_FRAC * (maxLat - minLat); + + // 2) ind × iteration 次模拟 + int totalRows = ind * iteration; + int[] indArr = new int[totalRows]; + long[][] out = new long[totalRows][cameraN]; + long[] count = new long[cameraN]; + + double biasRad = biasDeg / 360.0 * 2.0 * Math.PI; + + for (int ite = 1; ite <= iteration; ite++) { + for (int k = 1; k <= ind; k++) { + // 初始位置(中心区域随机) + double locX = RandomUtils.uniform(lonLo, lonHi); + double locY = RandomUtils.uniform(latLo, latHi); + double x0 = locX, y0 = locY; + + // 初始朝向 + double theta = RandomUtils.uniform(0.0, 2.0 * Math.PI); + Arrays.fill(count, 0L); + + for (int i = 1; i <= stepN; i++) { + // 相机检测判定 + for (int j = 0; j < cameraN; j++) { + double dx = locX - camLon[j]; + double dy = locY - camLat[j]; + double d = Math.sqrt(dx * dx + dy * dy); + if (d < detect) { + double bear; + if (dx == 0 && dy == 0) { + bear = 0; + } else { + bear = Math.atan2(dy, dx); + } + double diff = Math.abs(normalizeAngle(bear - bearing[j])); + if (diff < DETECT_HALF_ARC) { + count[j]++; + } + } + } + + // 步长 + double move = stepL * RandomUtils.gaussian(1.0, stepV); + locX += move * Math.cos(theta); + locY += move * Math.sin(theta); + + // 前进 vs 回原点 + double thetaF = theta + RandomUtils.gaussian(0.0, biasRad); + double thetaB; + if (locX > x0) { + thetaB = Math.PI + Math.atan2(locY - y0, locX - x0) + RandomUtils.gaussian(0.0, biasRad * 2); + } else { + thetaB = Math.atan2(locY - y0, locX - x0) + RandomUtils.gaussian(0.0, biasRad * 2); + } + double dist = Math.sqrt((locX - x0) * (locX - x0) + (locY - y0) * (locY - y0)); + double pForward = Math.abs(1.0 - dist / rangeM); + double pBackward = dist / rangeM; + theta = RandomUtils.weightedSample(new double[]{pForward, pBackward}) == 0 ? thetaF : thetaB; + } + + // 排序后写入(R 中 sort(camera$Count)) + long[] sorted = count.clone(); + Arrays.sort(sorted); + int rowIdx = k + (ite - 1) * ind - 1; + indArr[rowIdx] = k; + out[rowIdx] = sorted; + } + } + + return new SimulationResult(indArr, out, camLon, camLat); + } + + private static double[] uniqueLon(java.util.List data) { + java.util.LinkedHashMap map = new java.util.LinkedHashMap<>(); + for (CameraPoint p : data) { + map.put(p.getLon() + "," + p.getLat(), new double[]{p.getLon(), p.getLat()}); + } + double[] lons = new double[map.size()]; + double[] lats = new double[map.size()]; + int i = 0; + for (double[] v : map.values()) { + lons[i] = v[0]; + lats[i] = v[1]; + i++; + } + // 共享给 uniqueLat 通过线程局部?简单起见直接重写为返回 pair + return lons; + } + + private static double[] uniqueLat(java.util.List data) { + java.util.LinkedHashMap map = new java.util.LinkedHashMap<>(); + for (CameraPoint p : data) { + map.put(p.getLon() + "," + p.getLat(), new double[]{p.getLon(), p.getLat()}); + } + double[] lats = new double[map.size()]; + int i = 0; + for (double[] v : map.values()) { + lats[i] = v[1]; + i++; + } + return lats; + } + + /** 把角度规整到 [-π, π]。 */ + private static double normalizeAngle(double a) { + while (a > Math.PI) { + a -= 2.0 * Math.PI; + } + while (a < -Math.PI) { + a += 2.0 * Math.PI; + } + return a; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/controller/CameraTrapStatController.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/controller/CameraTrapStatController.java new file mode 100644 index 0000000..0508e99 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/controller/CameraTrapStatController.java @@ -0,0 +1,609 @@ +package net.hzvtc.stat.controller; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import net.hzvtc.stat.algorithm.Geo2Proj; +import net.hzvtc.stat.algorithm.PlotCamtrap; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.ChainLengthItem; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.DensitySeries; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.FootprintTrace; +import net.hzvtc.stat.dto.HabitatAreaResult; +import net.hzvtc.stat.dto.HabitatRangeResult; +import net.hzvtc.stat.dto.HomeRangeItem; +import net.hzvtc.stat.dto.MoveBiasResult; +import net.hzvtc.stat.dto.PopulationEstimateItem; +import net.hzvtc.stat.dto.PredictionResult; +import net.hzvtc.stat.dto.RasterResult; +import net.hzvtc.stat.dto.SimulationResult; +import net.hzvtc.stat.service.CameraTrapStatService; +import net.hzvtc.stat.service.StatEsDataService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * cameratrapR 11 个函数的 REST 入口(前端调用)。 + * + *

所有 endpoint 均接收 JSON body,数据来源二选一: + *

    + *
  • body 中直接传 data 数组(列字段见各方法注释,用于调试 / 自定义数据);
  • + *
  • 缺省 data 时按查询参数从 ES 索引 animal_valid_record 加载: + * 支持 species_id / speciesInIds / deviceIds / dateFrom / dateTo / checked / + * recordType / mediaType 等参数(与报表模块一致),经纬度由 device_id + * 关联 device_info 表补齐。
  • + *

+ * + *

返回 JSON 对象包含算法结果,便于前端直接渲染。

+ * + *

路径前缀:/es/stat

+ */ +@RestController +@RequestMapping("/es/stat") +@RequiredArgsConstructor +public class CameraTrapStatController { + + private final CameraTrapStatService service; + + private final StatEsDataService statEsDataService; + + /** + * 数据来源:body 中带非空 data 数组则直接使用(调试 / 自定义数据); + * 否则按查询参数(species_id / speciesInIds / deviceIds / dateFrom / dateTo / checked 等) + * 从 ES 索引 animal_valid_record 加载,并关联 device_info 补齐经纬度。 + */ + private List cameraData(JSONObject param) { + com.alibaba.fastjson.JSONArray data = param.getJSONArray("data"); + if (data != null && !data.isEmpty()) { + return CameraTrapStatService.toCameraPoints(data); + } + return statEsDataService.loadCameraPoints(param); + } + + private List footprintData(JSONObject param) { + com.alibaba.fastjson.JSONArray data = param.getJSONArray("data"); + if (data != null && !data.isEmpty()) { + return CameraTrapStatService.toFootprintPoints(data); + } + return statEsDataService.loadFootprintPoints(param); + } + + private List chainData(JSONObject param) { + com.alibaba.fastjson.JSONArray data = param.getJSONArray("data"); + if (data != null && !data.isEmpty()) { + return CameraTrapStatService.toChainRecords(data); + } + return statEsDataService.loadChainRecords(param); + } + + /** + * POST /es/stat/plotCamtrap + * body: { circleSize: 0.2, pointScatter: 5, data: [{lon,lat,groupSize,date,time}, ...] } + */ + @PostMapping("/plotCamtrap") + public JSONObject plotCamtrap(@RequestBody JSONObject param) { + double circleSize = getDouble(param, "circleSize", 0.2); + double pointScatter = getDouble(param, "pointScatter", 5.0); + List data = cameraData(param); + PlotCamtrap.Result result = service.plotCamtrap(data, circleSize, pointScatter); + + JSONObject out = new JSONObject(); + JSONArray agg = new JSONArray(); + for (CamtrapAggView p : toAggViews(result)) { + JSONObject o = new JSONObject(); + o.put("lat", p.lat); + o.put("lon", p.lon); + o.put("count", p.count); + agg.add(o); + } + JSONArray scatter = new JSONArray(); + result.getScatterPoints().forEach(p -> { + JSONObject o = new JSONObject(); + o.put("lon", p.getLon()); + o.put("lat", p.getLat()); + o.put("date", p.getDate()); + o.put("time", p.getTime()); + o.put("groupSize", p.getGroupSize()); + scatter.add(o); + }); + out.put("aggPoints", agg); + out.put("scatterPoints", scatter); + out.put("circleSize", result.getCircleSize()); + return out; + } + + /** + * POST /es/stat/dailyRhythm + * body: { data: [{lon,lat,groupSize,date,time}, ...] } + */ + @PostMapping("/dailyRhythm") + public JSONObject dailyRhythm(@RequestBody JSONObject param) { + List data = cameraData(param); + List series = service.dailyRhythm(data); + JSONObject out = new JSONObject(); + JSONArray arr = new JSONArray(); + for (DensitySeries s : series) { + JSONObject o = new JSONObject(); + o.put("x", s.getX()); + o.put("y", s.getY()); + o.put("bandwidth", s.getBandwidth()); + o.put("label", s.getLabel()); + arr.add(o); + } + out.put("series", arr); + return out; + } + + /** + * POST /es/stat/geo2proj + * body: { data: [{lon,lat,...}, ...] } + */ + @PostMapping("/geo2proj") + public JSONObject geo2proj(@RequestBody JSONObject param) { + List data = cameraData(param); + List utm = service.geo2proj(data); + JSONObject out = new JSONObject(); + JSONArray arr = new JSONArray(); + utm.forEach(p -> { + JSONObject o = new JSONObject(); + o.put("easting", p.getEasting()); + o.put("northing", p.getNorthing()); + arr.add(o); + }); + out.put("points", arr); + return out; + } + + /** + * POST /es/stat/simuCamtrap + * body: { species: "Red fox", ind: 10, iteration: 3, data: [...] } + */ + @PostMapping("/simuCamtrap") + public JSONObject simuCamtrap(@RequestBody JSONObject param) { + List data = cameraData(param); + String species = param.getString("species"); + int ind = getInt(param, "ind", 10); + int iteration = getInt(param, "iteration", 3); + SimulationResult res = service.simuCamtrap(data, species, ind, iteration); + JSONObject out = new JSONObject(); + out.put("ind", res.getInd()); + out.put("cameraCounts", res.getCameraCounts()); + out.put("cameraLon", res.getCameraLon()); + out.put("cameraLat", res.getCameraLat()); + return out; + } + + /** + * POST /es/stat/predictCamtrap + * body: { simu: {ind, cameraCounts, cameraLon, cameraLat}, data: [...], nTree: 1000, seed: 1234 } + */ + @PostMapping("/predictCamtrap") + public JSONObject predictCamtrap(@RequestBody JSONObject param) { + // 解析 simu + JSONObject simuObj = param.getJSONObject("simu"); + SimulationResult simu = new SimulationResult(); + JSONArray indArr = simuObj.getJSONArray("ind"); + int[] ind = new int[indArr.size()]; + for (int i = 0; i < ind.length; i++) { + ind[i] = indArr.getIntValue(i); + } + simu.setInd(ind); + + JSONArray counts = simuObj.getJSONArray("cameraCounts"); + long[][] cc = new long[counts.size()][]; + for (int i = 0; i < cc.length; i++) { + JSONArray row = counts.getJSONArray(i); + cc[i] = new long[row.size()]; + for (int j = 0; j < row.size(); j++) { + cc[i][j] = row.getLongValue(j); + } + } + simu.setCameraCounts(cc); + + JSONArray lonArr = simuObj.getJSONArray("cameraLon"); + JSONArray latArr = simuObj.getJSONArray("cameraLat"); + double[] lon = new double[lonArr.size()]; + double[] lat = new double[latArr.size()]; + for (int i = 0; i < lon.length; i++) { + lon[i] = lonArr.getDoubleValue(i); + lat[i] = latArr.getDoubleValue(i); + } + simu.setCameraLon(lon); + simu.setCameraLat(lat); + + List real = cameraData(param); + int nTree = getInt(param, "nTree", 1000); + long seed = getLong(param, "seed", 1234L); + + PredictionResult res = service.predictCamtrap(simu, real, nTree, seed); + JSONObject out = new JSONObject(); + out.put("noInd", res.getNoInd()); + out.put("ciLow", res.getCiLow()); + out.put("ciHigh", res.getCiHigh()); + return out; + } + + /** + * POST /es/stat/plotFootprint + * body: { data: [{lon,lat,date}, ...] } + */ + @PostMapping("/plotFootprint") + public JSONObject plotFootprint(@RequestBody JSONObject param) { + List chain = footprintData(param); + FootprintTrace trace = service.plotFootprint(chain); + JSONObject out = new JSONObject(); + JSONArray pts = new JSONArray(); + trace.getPoints().forEach(p -> { + JSONObject o = new JSONObject(); + o.put("lon", p.getLon()); + o.put("lat", p.getLat()); + o.put("date", p.getDate()); + pts.add(o); + }); + out.put("points", pts); + out.put("startLon", trace.getStartLon()); + out.put("startLat", trace.getStartLat()); + out.put("endLon", trace.getEndLon()); + out.put("endLat", trace.getEndLat()); + return out; + } + + /** + * POST /es/stat/moveBias + * body: { scale: 2, data: [{lon,lat,date}, ...] } + */ + @PostMapping("/moveBias") + public JSONObject moveBias(@RequestBody JSONObject param) { + List chain = footprintData(param); + int scale = getInt(param, "scale", 2); + MoveBiasResult res = service.moveBias(chain, scale); + JSONObject out = new JSONObject(); + out.put("meanStepLength", res.getMeanStepLength()); + out.put("sdStepLength", res.getSdStepLength()); + out.put("sdAngularDeflection", res.getSdAngularDeflection()); + return out; + } + + /** + * POST /es/stat/rowcliffe + * body: { r: 0.02, theta: 40, v: 2, duration: 40, data: [...] } + */ + @PostMapping("/rowcliffe") + public JSONObject rowcliffe(@RequestBody JSONObject param) { + List data = cameraData(param); + double r = getDouble(param, "r", 0.02); + double theta = getDouble(param, "theta", 40); + double v = getDouble(param, "v", 2); + double duration = getDouble(param, "duration", 40); + double[] res = service.rowcliffe(data, r, theta, v, duration); + JSONObject out = new JSONObject(); + out.put("density", res[0]); + out.put("sd", res[1]); + return out; + } + + /** + * POST /es/stat/homeRange + * body: { data: [{lon,lat,date}, ...] } + */ + @PostMapping("/homeRange") + public JSONObject homeRange(@RequestBody JSONObject param) { + List chain = footprintData(param); + double area = service.homeRange(chain); + JSONObject out = new JSONObject(); + out.put("area", area); + return out; + } + + /** + * POST /es/stat/homeRanges + * body: { data: [{species, line, lon, lat, date}, ...] } + */ + @PostMapping("/homeRanges") + public JSONObject homeRanges(@RequestBody JSONObject param) { + List recs = chainData(param); + List list = service.homeRanges(recs); + JSONObject out = new JSONObject(); + JSONArray arr = new JSONArray(); + list.forEach(it -> { + JSONObject o = new JSONObject(); + o.put("species", it.getSpecies()); + o.put("area", it.getArea()); + o.put("sd", it.getSd()); + o.put("n", it.getN()); + arr.add(o); + }); + out.put("items", arr); + return out; + } + + /** + * POST /es/stat/chainLength + * body: { data: [{lon,lat,date}, ...] } + */ + @PostMapping("/chainLength") + public JSONObject chainLength(@RequestBody JSONObject param) { + List chain = footprintData(param); + double dist = service.chainLength(chain); + JSONObject out = new JSONObject(); + out.put("distance", dist); + return out; + } + + /** + * POST /es/stat/chainLengths + * body: { data: [{species, line, lon, lat, date}, ...] } + */ + @PostMapping("/chainLengths") + public JSONObject chainLengths(@RequestBody JSONObject param) { + List recs = chainData(param); + List list = service.chainLengths(recs); + JSONObject out = new JSONObject(); + JSONArray arr = new JSONArray(); + list.forEach(it -> { + JSONObject o = new JSONObject(); + o.put("species", it.getSpecies()); + o.put("distance", it.getDistance()); + o.put("distSd", it.getDistSd()); + o.put("stepL", it.getStepL()); + o.put("stepNo", it.getStepNo()); + o.put("n", it.getN()); + arr.add(o); + }); + out.put("items", arr); + return out; + } + + // ===================================================================== + // 报表四步:种群大小估计 → 种群密度估计 → 栖息地面积分析 → 栖息地分布范围 + // ===================================================================== + + /** + * POST /es/stat/populationEstimate(报表步骤 1:种群大小估计) + * body: { speciesList: ["Red fox", ...](空 = 全部物种), ind: 10, iteration: 3, + * nTree: 500, seed: 1234, data: [{lon,lat,groupSize,date,time,species}, ...] 或 ES 查询参数 } + * 返回 items: [{species, noInd, ciLow, ciHigh, individual[], recordCount, cameraCount}] + */ + @PostMapping("/populationEstimate") + public JSONObject populationEstimate(@RequestBody JSONObject param) { + List speciesList = new java.util.ArrayList<>(); + JSONArray spArr = param.getJSONArray("speciesList"); + if (spArr != null) { + for (int i = 0; i < spArr.size(); i++) { + speciesList.add(spArr.getString(i)); + } + } + List data = cameraData(param); + int ind = getInt(param, "ind", 10); + int iteration = getInt(param, "iteration", 3); + int nTree = getInt(param, "nTree", 500); + long seed = getLong(param, "seed", 1234L); + List items = service.populationEstimate(data, speciesList, ind, iteration, nTree, seed); + + JSONObject out = new JSONObject(); + JSONArray arr = new JSONArray(); + for (PopulationEstimateItem it : items) { + JSONObject o = new JSONObject(); + o.put("species", it.getSpecies()); + o.put("speciesName", it.getSpeciesName()); + o.put("noInd", it.getNoInd()); + o.put("ciLow", it.getCiLow()); + o.put("ciHigh", it.getCiHigh()); + JSONArray indArr = new JSONArray(); + if (it.getIndividual() != null) { + for (double v : it.getIndividual()) { + indArr.add(v); + } + } + o.put("individual", indArr); + o.put("recordCount", it.getRecordCount()); + o.put("cameraCount", it.getCameraCount()); + arr.add(o); + } + out.put("items", arr); + return out; + } + + /** + * POST /es/stat/densityEstimate(报表步骤 2:种群密度估计) + * body: { cellSize: 0.01, nTree: 200, seed: 1234, data: [...] 或 ES 查询参数 } + * 返回栅格 {nx, ny, x0, y0, cellSize, values[], maxValue, cameraCount, photoCount} + */ + @PostMapping("/densityEstimate") + public JSONObject densityEstimate(@RequestBody JSONObject param) { + List data = cameraData(param); + double cellSize = getDouble(param, "cellSize", 0.01); + int nTree = getInt(param, "nTree", 200); + long seed = getLong(param, "seed", 1234L); + RasterResult raster = service.densityEstimate(data, cellSize, nTree, seed); + return rasterToJson(raster); + } + + /** + * POST /es/stat/habitatArea(报表步骤 3:栖息地面积分析) + * body: { threshold: 0(≤0 自动 max-Kappa), raster: {nx,ny,x0,y0,cellSize,values}, + * data: [...](相机点位 = 出现样点,可用 ES 查询参数代替)} + * 返回 {threshold, method, suitableCells, totalCells, areaKm2, binary[], ...栅格几何} + */ + @PostMapping("/habitatArea") + public JSONObject habitatArea(@RequestBody JSONObject param) { + RasterResult raster = parseRaster(param.getJSONObject("raster")); + List presence = cameraData(param); + double manual = getDouble(param, "threshold", 0.0); + HabitatAreaResult res = service.habitatArea(raster, presence, manual); + + JSONObject out = new JSONObject(); + out.put("threshold", res.getThreshold()); + out.put("method", res.getMethod()); + out.put("suitableCells", res.getSuitableCells()); + out.put("totalCells", res.getTotalCells()); + out.put("areaKm2", res.getAreaKm2()); + out.put("nx", res.getNx()); + out.put("ny", res.getNy()); + out.put("x0", res.getX0()); + out.put("y0", res.getY0()); + out.put("cellSize", res.getCellSize()); + out.put("maxValue", res.getMaxValue()); + JSONArray bin = new JSONArray(); + if (res.getBinary() != null) { + for (int v : res.getBinary()) { + bin.add(v); + } + } + out.put("binary", bin); + return out; + } + + /** + * POST /es/stat/habitatRange(报表步骤 4:栖息地分布范围) + * body: { habitat: {nx,ny,x0,y0,cellSize,suitableCells,binary[]} }(步骤 3 输出) + * 返回 {polygon: [[lon,lat],...], areaKm2, suitableCells, totalCells, suitableRatio} + */ + @PostMapping("/habitatRange") + public JSONObject habitatRange(@RequestBody JSONObject param) { + JSONObject h = param.getJSONObject("habitat"); + if (h == null) { + throw new IllegalArgumentException("缺少 habitat 参数(应为 habitatArea 的输出)"); + } + HabitatAreaResult area = new HabitatAreaResult(); + area.setNx(h.getIntValue("nx")); + area.setNy(h.getIntValue("ny")); + area.setX0(h.getDoubleValue("x0")); + area.setY0(h.getDoubleValue("y0")); + area.setCellSize(h.getDoubleValue("cellSize")); + area.setSuitableCells(h.getIntValue("suitableCells")); + JSONArray binArr = h.getJSONArray("binary"); + int[] binary = new int[binArr == null ? 0 : binArr.size()]; + if (binArr != null) { + for (int i = 0; i < binary.length; i++) { + binary[i] = binArr.getIntValue(i); + } + } + area.setBinary(binary); + + HabitatRangeResult res = service.habitatRange(area); + + JSONObject out = new JSONObject(); + JSONArray poly = new JSONArray(); + if (res.getPolygon() != null) { + for (double[] p : res.getPolygon()) { + JSONArray pt = new JSONArray(); + pt.add(p[0]); + pt.add(p[1]); + poly.add(pt); + } + } + // 各连通斑块凸包(按面积降序) + JSONArray polys = new JSONArray(); + if (res.getPolygons() != null) { + for (double[][] ring : res.getPolygons()) { + JSONArray r = new JSONArray(); + if (ring != null) { + for (double[] p : ring) { + JSONArray pt = new JSONArray(); + pt.add(p[0]); + pt.add(p[1]); + r.add(pt); + } + } + polys.add(r); + } + } + out.put("polygon", poly); + out.put("polygons", polys); + out.put("clusterCount", res.getClusterCount()); + out.put("areaKm2", res.getAreaKm2()); + out.put("suitableCells", res.getSuitableCells()); + out.put("totalCells", res.getTotalCells()); + out.put("suitableRatio", res.getSuitableRatio()); + return out; + } + + /** RasterResult → JSON。 */ + private static JSONObject rasterToJson(RasterResult r) { + JSONObject o = new JSONObject(); + o.put("nx", r.getNx()); + o.put("ny", r.getNy()); + o.put("x0", r.getX0()); + o.put("y0", r.getY0()); + o.put("cellSize", r.getCellSize()); + o.put("maxValue", r.getMaxValue()); + o.put("cameraCount", r.getCameraCount()); + o.put("photoCount", r.getPhotoCount()); + JSONArray vals = new JSONArray(); + if (r.getValues() != null) { + for (double v : r.getValues()) { + vals.add(v); + } + } + o.put("values", vals); + return o; + } + + /** 前端传入栅格 JSON → RasterResult。 */ + private static RasterResult parseRaster(JSONObject r) { + if (r == null) { + throw new IllegalArgumentException("缺少 raster 参数(应为 densityEstimate 的输出)"); + } + RasterResult raster = new RasterResult(); + raster.setNx(r.getIntValue("nx")); + raster.setNy(r.getIntValue("ny")); + raster.setX0(r.getDoubleValue("x0")); + raster.setY0(r.getDoubleValue("y0")); + raster.setCellSize(r.getDoubleValue("cellSize")); + raster.setMaxValue(r.getDoubleValue("maxValue")); + JSONArray vals = r.getJSONArray("values"); + double[] values = new double[vals == null ? 0 : vals.size()]; + if (vals != null) { + for (int i = 0; i < values.length; i++) { + values[i] = vals.getDoubleValue(i); + } + } + raster.setValues(values); + return raster; + } + + // ===================================================================== + // fastjson 1.x 兼容辅助(getDoubleValue(key, def) 等带默认值重载是 fastjson2 API) + // ===================================================================== + + private static double getDouble(JSONObject o, String key, double def) { + Double v = o.getDouble(key); + return v == null ? def : v; + } + + private static int getInt(JSONObject o, String key, int def) { + Integer v = o.getInteger(key); + return v == null ? def : v; + } + + private static long getLong(JSONObject o, String key, long def) { + Long v = o.getLong(key); + return v == null ? def : v; + } + + /** 聚合点位视图(避免在 lambda 里再构造 JSONObject 嵌套)。 */ + private static List toAggViews(PlotCamtrap.Result result) { + List views = new java.util.ArrayList<>(result.getAggPoints().size()); + result.getAggPoints().forEach(p -> views.add(new CamtrapAggView(p.getLat(), p.getLon(), p.getCount()))); + return views; + } + + private static final class CamtrapAggView { + private final double lat; + private final double lon; + private final long count; + + private CamtrapAggView(double lat, double lon, long count) { + this.lat = lat; + this.lon = lon; + this.count = count; + } + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/domain/SpeciesMovementParam.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/domain/SpeciesMovementParam.java new file mode 100644 index 0000000..84f068d --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/domain/SpeciesMovementParam.java @@ -0,0 +1,13 @@ +package net.hzvtc.stat.domain; + +import lombok.Data; + +/** Camera-trap movement parameters loaded from MySQL. */ +@Data +public class SpeciesMovementParam { + + private double stepL; + private double bias; + private double rangeKm2; + private int ind; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CameraPoint.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CameraPoint.java new file mode 100644 index 0000000..829a052 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CameraPoint.java @@ -0,0 +1,46 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * cameratrapR 中 trapresult 数据框的 Java 表示(plotCamtrap / Rowcliffe / simuCamtrap / predictCamtrap 的输入)。 + * + *

对应 R data.frame 列:Lon、Lat、Group_size、Date、Time。

+ * + *

各字段语义: + *

    + *
  • Lon / Lat:相机拍摄位点的经纬度(度,WGS84)
  • + *
  • GroupSize:本条记录对应物种的群体大小(每张照片/视频中个体数)
  • + *
  • Date:拍摄日期(yyyy-MM-dd)
  • + *
  • Time:拍摄时间(HH:mm:ss)
  • + *
+ *

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CameraPoint { + + /** 经度(度)。 */ + private double lon; + + /** 纬度(度)。 */ + private double lat; + + /** 群体大小(单张记录中的个体数)。 */ + private int groupSize; + + /** 日期,字符串 yyyy-MM-dd。 */ + private String date; + + /** 时间,字符串 HH:mm:ss。 */ + private String time; + + /** 物种名(ES scientific_name / CSV species 列;多物种统计与密度估计用,可为空)。 */ + private String species; + + /** 物种中文名(ES species_name / CSV speciesName 列;物种过滤时与学名共同参与匹配,可为空)。 */ + private String speciesName; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapAggPoint.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapAggPoint.java new file mode 100644 index 0000000..a455953 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapAggPoint.java @@ -0,0 +1,25 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * plotCamtrap 输出:按 (Lat, Lon) 聚合后的相机点位(每个点位上的照片总次数 Count)。 + * + *

对应 R 中 sum = aggregate(x$Group_size, by=list(x$Lat,x$Lon), sum) 的输出。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CamtrapAggPoint { + + /** 纬度(度)。 */ + private double lat; + + /** 经度(度)。 */ + private double lon; + + /** 该点位累计的 GroupSize 总和(用于决定绘图时圆点大小:cex=Count*circle.size)。 */ + private long count; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapScatter.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapScatter.java new file mode 100644 index 0000000..dba8c43 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/CamtrapScatter.java @@ -0,0 +1,33 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * plotCamtrap 输出:单条记录的散点(jitter 后)坐标。 + * + *

对应 R 中 jitter(X$Lon, factor=point.scatter) / jitter(X$Lat, factor=point.scatter)。

+ * + *

每个原始记录(Group_size > 0)都会产出一个 jitter 后坐标点;前端画散点。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CamtrapScatter { + + /** jitter 后的经度。 */ + private double lon; + + /** jitter 后的纬度。 */ + private double lat; + + /** 原始日期(用于按时间上色)。 */ + private String date; + + /** 原始时间。 */ + private String time; + + /** 原始群体大小。 */ + private int groupSize; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainLengthItem.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainLengthItem.java new file mode 100644 index 0000000..8d29d53 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainLengthItem.java @@ -0,0 +1,31 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * chainLengths 输出行:每个物种的足迹链长度汇总。 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ChainLengthItem { + + private String species; + + /** 平均链长(km)。 */ + private double distance; + + /** 标准差。 */ + private double distSd; + + /** 平均步长(m)。 */ + private double stepL; + + /** 推算步数(30 天)。 */ + private long stepNo; + + /** 链样本数。 */ + private int n; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainRecord.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainRecord.java new file mode 100644 index 0000000..701f823 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/ChainRecord.java @@ -0,0 +1,27 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * chainDB 行:单条足迹链记录,附 Species、Line 字段,用于多链多物种聚合。 + * + *

对应 R 中 chainDB 数据框(multi-species homeRanges / chainLengths 的输入)。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ChainRecord { + + private String species; + + /** 链编号(同物种可有多条链)。 */ + private int line; + + private double lon; + private double lat; + + /** 记录时间字符串。 */ + private String date; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/DensitySeries.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/DensitySeries.java new file mode 100644 index 0000000..f267af0 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/DensitySeries.java @@ -0,0 +1,37 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 核密度估计(KDE)曲线数据:x[] / y[] 等长数组,用于前端画 density 曲线。 + * + *

对应 R 中 density(times, bw) 返回对象的可视化数据。

+ * + *

字段语义: + *

    + *
  • x:采样点序列(升序)
  • + *
  • y:在 x 处的密度值
  • + *
  • bandwidth:核密度带宽 bw(参考输出,便于调试)
  • + *
  • label:曲线标签(如 "bw=0.3"、"bw=2")
  • + *
+ *

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class DensitySeries { + + /** x 采样点序列(升序)。 */ + private double[] x; + + /** 密度值序列,与 x 等长。 */ + private double[] y; + + /** 核密度带宽(bandwidth)。 */ + private double bandwidth; + + /** 曲线标签(用于前端 legend)。 */ + private String label; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintPoint.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintPoint.java new file mode 100644 index 0000000..91d6d07 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintPoint.java @@ -0,0 +1,22 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * plotFootprint 输入 / 输出元素:单个足迹链节点(Lon, Lat, Date)。 + * + *

R 中 chain 数据框列:Lon、Lat、Date(按时间排序、采样间隔约 1 秒)。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FootprintPoint { + + private double lon; + private double lat; + + /** 记录时间,字符串 yyyy-MM-dd HH:mm:ss。 */ + private String date; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintTrace.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintTrace.java new file mode 100644 index 0000000..4322176 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/FootprintTrace.java @@ -0,0 +1,27 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * plotFootprint 输出:折线轨迹 + 起止点高亮。 + * + *

前端可用 points 顺序连线,并在 startLon, startLat(蓝色三角)与 + * endLon, endLat(红色方块)画标记。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FootprintTrace { + + /** 折线轨迹(按时间顺序)。 */ + private List points; + + private double startLon; + private double startLat; + private double endLon; + private double endLat; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatAreaResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatAreaResult.java new file mode 100644 index 0000000..db1b52f --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatAreaResult.java @@ -0,0 +1,46 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 栖息地面积分析输出:基于最优阈值的适宜生境二值栅格。 + * + *

阈值默认用 max-Kappa 法(以相机所在格为出现样点、背景格为未出现样点, + * 遍历候选阈值取 Cohen's Kappa 最大者),每个物种因密度栅格不同而阈值不同; + * 也可由前端显式传入阈值覆盖。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class HabitatAreaResult { + + /** 采用的阈值(密度值 ≥ 阈值视为适宜生境)。 */ + private double threshold; + + /** 阈值确定方法:maxKappa / manual。 */ + private String method; + + /** 适宜生境格数。 */ + private int suitableCells; + + /** 总格数。 */ + private int totalCells; + + /** 适宜生境面积(km²,按格面积 × 格数,格面积随纬度校正)。 */ + private double areaKm2; + + /** ---- 栅格几何(与输入密度栅格一致,供下一步分布范围使用)---- */ + private int nx; + private int ny; + private double x0; + private double y0; + private double cellSize; + + /** 二值栅格(1 = 适宜生境,0 = 不适宜;行优先,row 0 最南)。 */ + private int[] binary; + + /** 透传的源密度栅格最大值(前端图例用)。 */ + private double maxValue; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatRangeResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatRangeResult.java new file mode 100644 index 0000000..296a076 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HabitatRangeResult.java @@ -0,0 +1,36 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 栖息地分布范围输出:按连通斑块分别求最小凸包(不再跨斑块合并, + * 避免散布斑块之间的空隙被单一凸包圈入导致范围虚高)。 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class HabitatRangeResult { + + /** 最大斑块凸包多边形顶点(闭合环,最后一个点 = 第一个点),[i][0]=lon,[i][1]=lat。 */ + private double[][] polygon; + + /** 各连通斑块凸包多边形(按面积降序;polygon = 其中最大者)。 */ + private double[][][] polygons; + + /** 连通斑块数。 */ + private int clusterCount; + + /** 全部斑块凸包面积之和(km²)。 */ + private double areaKm2; + + /** 适宜生境格数(输入二值栅格中 =1 的格子数)。 */ + private int suitableCells; + + /** 总格数。 */ + private int totalCells; + + /** 栖息地面积占分布范围面积的比例(0-1)。 */ + private double suitableRatio; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HomeRangeItem.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HomeRangeItem.java new file mode 100644 index 0000000..6310033 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/HomeRangeItem.java @@ -0,0 +1,25 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * homeRanges 输出行:每个物种的家域面积(km²)。 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class HomeRangeItem { + + private String species; + + /** 平均家域面积(km²)。 */ + private double area; + + /** 标准差。 */ + private double sd; + + /** 链样本数。 */ + private int n; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/MoveBiasResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/MoveBiasResult.java new file mode 100644 index 0000000..ab07c77 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/MoveBiasResult.java @@ -0,0 +1,25 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * moveBias 输出:步长统计量与转角统计量。 + * + *

对应 R 返回:c(mean(dist), sd(dist), sd(delta.th))。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class MoveBiasResult { + + /** 步长均值(m)。 */ + private double meanStepLength; + + /** 步长标准差(m)。 */ + private double sdStepLength; + + /** 转向角标准差(度)。 */ + private double sdAngularDeflection; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PopulationEstimateItem.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PopulationEstimateItem.java new file mode 100644 index 0000000..1e38404 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PopulationEstimateItem.java @@ -0,0 +1,41 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * populationEstimate 输出项:单物种的种群大小(数量)估计。 + * + *

对应 R 中 simuCamtrap + predictCamtrap 组合的输出: + * No_ind(估计值)、CI_L(区间下界)、CI_U(区间上界)。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class PopulationEstimateItem { + + /** 物种名(ES scientific_name / species_name,CSV species 列;无物种信息时为 "全部")。 */ + private String species; + + /** 物种中文名(ES species_name;无中文名时回退为 species)。 */ + private String speciesName; + + /** 估计个体数 No_ind(随机森林树预测中位数)。 */ + private double noInd; + + /** 95% 置信区间下界 CI_L。 */ + private double ciLow; + + /** 95% 置信区间上界 CI_U。 */ + private double ciHigh; + + /** 每棵树的预测值采样(≤200,供箱线图)。 */ + private double[] individual; + + /** 参与计算的记录数。 */ + private int recordCount; + + /** 参与计算的相机点位数。 */ + private int cameraCount; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PredictionResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PredictionResult.java new file mode 100644 index 0000000..d6b7ecb --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/PredictionResult.java @@ -0,0 +1,30 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * predictCamtrap 输出:随机森林 1000 棵树的预测中位数与 95% 置信区间。 + * + *

对应 R 返回:data.frame(No_ind, CI_L, CI_U)。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class PredictionResult { + + /** 估计的个体数(中位数)。 */ + private double noInd; + + /** 95% 置信区间下界。 */ + private double ciLow; + + /** 95% 置信区间上界。 */ + private double ciHigh; + + /** + * 每棵树的预测个体数(供前端画箱线图;最多 200 个采样值,中位数/CI 仍基于全量树)。 + */ + private double[] individual; +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/RasterResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/RasterResult.java new file mode 100644 index 0000000..f08f25a --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/RasterResult.java @@ -0,0 +1,63 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 种群密度估计(随机森林物种分布模型)输出的栅格结果。 + * + *

栅格以相机点位分布范围为边界(四周外扩 3 个格宽),行优先存储: + * values[row * nx + col],row 0 = 最南行,col 0 = 最西列。 + * 每格值域 [0, 1],表示该物种出现在该格子的概率。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RasterResult { + + /** 列数(西 → 东)。 */ + private int nx; + + /** 行数(南 → 北)。 */ + private int ny; + + /** 西南角经度(度)。 */ + private double x0; + + /** 西南角纬度(度)。 */ + private double y0; + + /** 格宽(度,经纬度同值)。 */ + private double cellSize; + + /** 每格出现概率(长度 = nx * ny,行优先,row 0 最南)。 */ + private double[] values; + + /** 栅格中的最大概率值。 */ + private double maxValue; + + /** 参与建模的相机点位数。 */ + private int cameraCount; + + /** 参与建模的照片(记录)总数。 */ + private long photoCount; + + /** 格子中心经度(度)。 */ + public double cellCenterLon(int col) { + return x0 + (col + 0.5) * cellSize; + } + + /** 格子中心纬度(度)。 */ + public double cellCenterLat(int row) { + return y0 + (row + 0.5) * cellSize; + } + + /** 指定格子的值(越界返回 0)。 */ + public double valueAt(int row, int col) { + if (row < 0 || row >= ny || col < 0 || col >= nx) { + return 0.0; + } + return values[row * nx + col]; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/SimulationResult.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/SimulationResult.java new file mode 100644 index 0000000..0569981 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/dto/SimulationResult.java @@ -0,0 +1,27 @@ +package net.hzvtc.stat.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * simuCamtrap 输出:模拟结果矩阵。 + * + *

对应 R 返回:data.frame(Ind, [每台相机的 Count])。 + * 行 = ind*iteration 次模拟;列数 = 相机数 + 1(第一列为个体数 Ind)。

+ */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SimulationResult { + + /** 每行的个体数(长度 = ind*iteration)。 */ + private int[] ind; + + /** 每个相机位的拍照次数(外层长度 = 行数;内层长度 = 相机数)。 */ + private long[][] cameraCounts; + + /** 相机坐标(与 cameraCounts 列对应)。 */ + private double[] cameraLon; + private double[] cameraLat; +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/mapper/SpeciesMovementParamMapper.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/mapper/SpeciesMovementParamMapper.java new file mode 100644 index 0000000..5c16e32 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/mapper/SpeciesMovementParamMapper.java @@ -0,0 +1,21 @@ +package net.hzvtc.stat.mapper; + +import net.hzvtc.stat.domain.SpeciesMovementParam; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +@Mapper +public interface SpeciesMovementParamMapper { + + @Select("SELECT step_length AS stepL, bias_degree AS bias, range_km2 AS rangeKm2, " + + "max_individuals AS ind FROM camera_trap_species_param " + + "WHERE enabled = 1 AND (species_name = #{species} OR scientific_name = #{species} " + + "OR chinese_name = #{species}) LIMIT 1") + SpeciesMovementParam selectBySpecies(@Param("species") String species); + + @Select("SELECT step_length AS stepL, bias_degree AS bias, range_km2 AS rangeKm2, " + + "max_individuals AS ind FROM camera_trap_species_param " + + "WHERE enabled = 1 AND species_name = '__DEFAULT__' LIMIT 1") + SpeciesMovementParam selectDefault(); +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/CameraTrapStatService.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/CameraTrapStatService.java new file mode 100644 index 0000000..d4664f6 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/CameraTrapStatService.java @@ -0,0 +1,131 @@ +package net.hzvtc.stat.service; + +import com.alibaba.fastjson.JSONArray; +import net.hzvtc.stat.algorithm.Geo2Proj; +import net.hzvtc.stat.algorithm.PlotCamtrap; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.ChainLengthItem; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.DensitySeries; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.FootprintTrace; +import net.hzvtc.stat.dto.HabitatAreaResult; +import net.hzvtc.stat.dto.HabitatRangeResult; +import net.hzvtc.stat.dto.HomeRangeItem; +import net.hzvtc.stat.dto.MoveBiasResult; +import net.hzvtc.stat.dto.PopulationEstimateItem; +import net.hzvtc.stat.dto.PredictionResult; +import net.hzvtc.stat.dto.RasterResult; +import net.hzvtc.stat.dto.SimulationResult; + +import java.util.List; + +/** + * cameratrapR 11 个函数的业务接口。 + * + *

实现见 {@link net.hzvtc.stat.service.impl.CameraTrapStatServiceImpl}。

+ */ +public interface CameraTrapStatService { + + // ============================== plotCamtrap ============================== + + PlotCamtrap.Result plotCamtrap(List data, double circleSize, double pointScatter); + + // ============================== dailyRhythm ============================== + + List dailyRhythm(List data); + + // ============================== geo2proj ============================== + + List geo2proj(List data); + + List geo2projFootprint(List data); + + // ============================== simuCamtrap ============================== + + SimulationResult simuCamtrap(List data, String species, int ind, int iteration); + + /** 自定义运动参数版本(保留 R 全部参数可调)。 */ + SimulationResult simuCamtrapCustom(List data, String species, + double detect, double[] bearing, + int stepN, double stepV, + double stepL, double biasDeg, + double rangeM, int ind, int iteration); + + // ============================== predictCamtrap ============================== + + PredictionResult predictCamtrap(SimulationResult simu, List realData, + int nTree, long seed); + + // ============================== populationEstimate(报表:种群大小估计) ============================== + + /** + * 多物种种群大小(数量)估计:按 species 分组逐物种执行 simuCamtrap + predictCamtrap。 + * + * @param data 相机记录(含 species 字段) + * @param speciesList 物种过滤(空 = 数据中出现的全部物种) + * @param ind 单物种模拟最大个体数 + * @param iteration 模拟迭代次数 + * @param nTree 随机森林树数 + * @param seed 随机种子 + */ + List populationEstimate(List data, List speciesList, + int ind, int iteration, int nTree, long seed); + + // ============================== densityEstimate(报表:种群密度估计) ============================== + + /** 随机森林 SDM 栅格化出现概率(输入:相机点位 + 积累照片总数)。 */ + RasterResult densityEstimate(List data, double cellSize, int nTree, long seed); + + // ============================== habitatArea(报表:栖息地面积分析) ============================== + + /** 密度栅格 + 最优阈值 → 适宜生境二值栅格(manual ≤ 0 时自动 max-Kappa)。 */ + HabitatAreaResult habitatArea(RasterResult raster, List presence, double manual); + + // ============================== habitatRange(报表:栖息地分布范围) ============================== + + /** 适宜生境二值栅格 → 最小凸包分布范围多边形。 */ + HabitatRangeResult habitatRange(HabitatAreaResult area); + + // ============================== plotFootprint ============================== + + FootprintTrace plotFootprint(List chain); + + // ============================== moveBias ============================== + + MoveBiasResult moveBias(List chain, int scale); + + // ============================== Rowcliffe ============================== + + double[] rowcliffe(List data, double r, double theta, double v, double duration); + + // ============================== homeRange(s) ============================== + + double homeRange(List chain); + + List homeRanges(List records); + + // ============================== chainLength(s) ============================== + + double chainLength(List chain); + + List chainLengths(List records); + + // ===================================================================== + // DTO 转换:JSONArray → List / ChainRecord / FootprintPoint + // (静态方法委托给实现类,调用方无需感知 Impl) + // ===================================================================== + + /** 前端传入 JSONArray(每项含 lon/lat/groupSize/date/time)→ CameraPoint 列表。 */ + static List toCameraPoints(JSONArray arr) { + return net.hzvtc.stat.service.impl.CameraTrapStatServiceImpl.toCameraPoints(arr); + } + + static List toFootprintPoints(JSONArray arr) { + return net.hzvtc.stat.service.impl.CameraTrapStatServiceImpl.toFootprintPoints(arr); + } + + static List toChainRecords(JSONArray arr) { + return net.hzvtc.stat.service.impl.CameraTrapStatServiceImpl.toChainRecords(arr); + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/StatEsDataService.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/StatEsDataService.java new file mode 100644 index 0000000..7230257 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/StatEsDataService.java @@ -0,0 +1,30 @@ +package net.hzvtc.stat.service; + +import com.alibaba.fastjson.JSONObject; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.FootprintPoint; + +import java.util.List; + +/** + * 从 ES 索引 animal_valid_record 加载数据的业务接口。 + * + *

实现见 {@link net.hzvtc.stat.service.impl.StatEsDataServiceImpl}。 + * 记录关联 device_info 表补齐设备经纬度后,转换为 stat 算法所需 DTO。

+ * + *

查询参数复用 CommonEsUtils.createQuery,支持: + * species_id / speciesInIds / deviceIds / dateFrom / dateTo / checked / + * recordType / mediaType / mediaTypes 等(与报表模块参数语义一致)。

+ */ +public interface StatEsDataService { + + /** 加载相机记录(plotCamtrap / dailyRhythm / geo2proj / simuCamtrap / rowcliffe 输入)。 */ + List loadCameraPoints(JSONObject param); + + /** 加载足迹链节点(plotFootprint / moveBias / homeRange / chainLength 输入),按时间升序。 */ + List loadFootprintPoints(JSONObject param); + + /** 加载多链记录(homeRanges / chainLengths 输入),按 species、line、时间排序。 */ + List loadChainRecords(JSONObject param); +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/CameraTrapStatServiceImpl.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/CameraTrapStatServiceImpl.java new file mode 100644 index 0000000..0d435d9 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/CameraTrapStatServiceImpl.java @@ -0,0 +1,330 @@ +package net.hzvtc.stat.service.impl; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import net.hzvtc.stat.algorithm.ChainLength; +import net.hzvtc.stat.algorithm.DailyRhythm; +import net.hzvtc.stat.algorithm.DensityEstimate; +import net.hzvtc.stat.algorithm.Geo2Proj; +import net.hzvtc.stat.algorithm.HabitatAnalysis; +import net.hzvtc.stat.algorithm.HabitatRange; +import net.hzvtc.stat.algorithm.HomeRange; +import net.hzvtc.stat.algorithm.MoveBias; +import net.hzvtc.stat.algorithm.PlotCamtrap; +import net.hzvtc.stat.algorithm.PlotFootprint; +import net.hzvtc.stat.algorithm.PredictCamtrap; +import net.hzvtc.stat.algorithm.Rowcliffe; +import net.hzvtc.stat.algorithm.SimuCamtrap; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.ChainLengthItem; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.DensitySeries; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.dto.FootprintTrace; +import net.hzvtc.stat.dto.HabitatAreaResult; +import net.hzvtc.stat.dto.HabitatRangeResult; +import net.hzvtc.stat.dto.HomeRangeItem; +import net.hzvtc.stat.dto.MoveBiasResult; +import net.hzvtc.stat.dto.PopulationEstimateItem; +import net.hzvtc.stat.dto.PredictionResult; +import net.hzvtc.stat.dto.RasterResult; +import net.hzvtc.stat.dto.SimulationResult; +import net.hzvtc.stat.domain.SpeciesMovementParam; +import net.hzvtc.stat.mapper.SpeciesMovementParamMapper; +import net.hzvtc.stat.service.CameraTrapStatService; +import net.hzvtc.stat.util.RandomUtils; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * {@link CameraTrapStatService} 实现:调用统计模型,并从 MySQL 加载物种运动参数。 + */ +@Service +@RequiredArgsConstructor +public class CameraTrapStatServiceImpl implements CameraTrapStatService { + + private final SpeciesMovementParamMapper speciesMovementParamMapper; + + // ============================== plotCamtrap ============================== + + @Override + public PlotCamtrap.Result plotCamtrap(List data, double circleSize, double pointScatter) { + return PlotCamtrap.plotCamtrap(data, circleSize, pointScatter); + } + + // ============================== dailyRhythm ============================== + + @Override + public List dailyRhythm(List data) { + return DailyRhythm.dailyRhythm(data); + } + + // ============================== geo2proj ============================== + + @Override + public List geo2proj(List data) { + return Geo2Proj.convertToDto(data); + } + + @Override + public List geo2projFootprint(List data) { + double[][] xy = Geo2Proj.convertFootprint(data); + List out = new ArrayList<>(xy.length); + for (double[] v : xy) { + out.add(new Geo2Proj.UtmPoint(v[0], v[1])); + } + return out; + } + + // ============================== simuCamtrap ============================== + + @Override + public SimulationResult simuCamtrap(List data, String species, int ind, int iteration) { + RandomUtils.setSeed(RandomUtils.DEFAULT_SEED); + return simulateBySpecies(data, species, ind, iteration); + } + + @Override + public SimulationResult simuCamtrapCustom(List data, String species, + double detect, double[] bearing, + int stepN, double stepV, + double stepL, double biasDeg, + double rangeM, int ind, int iteration) { + return SimuCamtrap.simuCamtrap(data, species, detect, bearing, stepN, stepV, + stepL, biasDeg, rangeM, ind, iteration); + } + + // ============================== predictCamtrap ============================== + + @Override + public PredictionResult predictCamtrap(SimulationResult simu, List realData, + int nTree, long seed) { + return PredictCamtrap.predictCamtrap(simu, realData, nTree, seed); + } + + // ============================== populationEstimate ============================== + + @Override + public List populationEstimate(List data, List speciesList, + int ind, int iteration, int nTree, long seed) { + List items = new ArrayList<>(); + if (data == null || data.isEmpty()) { + return items; + } + + // 按 species 分组(保持数据出现顺序) + Map> groups = new LinkedHashMap<>(); + for (CameraPoint p : data) { + String sp = p.getSpecies() == null ? "" : p.getSpecies().trim(); + groups.computeIfAbsent(sp, k -> new ArrayList<>()).add(p); + } + + // 物种过滤:学名(species)与中文名(speciesName)共同参与匹配(忽略大小写) + Set filter = null; + if (speciesList != null) { + filter = new LinkedHashSet<>(); + for (String s : speciesList) { + if (s != null && !s.trim().isEmpty()) { + filter.add(s.trim()); + } + } + } + + for (Map.Entry> e : groups.entrySet()) { + if (filter != null && !filter.isEmpty() && !matchSpecies(e.getValue(), filter)) { + continue; + } + List group = e.getValue(); + if (group.isEmpty()) { + continue; + } + + CameraPoint first = group.get(0); + + // 每个物种用独立派生种子重置模拟随机源(等价 R set.seed),保证同条件结果可复现 + RandomUtils.setSeed(seed * 1_000_003L + e.getKey().hashCode()); + SimulationResult simu = simulateBySpecies(group, e.getKey(), ind, iteration); + PredictionResult pred = PredictCamtrap.predictCamtrap(simu, group, nTree, seed); + + // 中文名:取组内第一条记录的 species_name,无则回退为学名 + String spName = first.getSpeciesName(); + String speciesKey = e.getKey().isEmpty() ? "全部" : e.getKey(); + if (spName == null || spName.trim().isEmpty()) { + spName = speciesKey; + } + + int cameraN = simu.getCameraLon() == null ? 0 : simu.getCameraLon().length; + items.add(new PopulationEstimateItem( + speciesKey, spName, + pred.getNoInd(), pred.getCiLow(), pred.getCiHigh(), + pred.getIndividual(), group.size(), cameraN)); + } + return items; + } + + private SimulationResult simulateBySpecies(List data, String species, int ind, int iteration) { + SpeciesMovementParam param = null; + if (species != null && !species.trim().isEmpty()) { + param = speciesMovementParamMapper.selectBySpecies(species.trim()); + } + if (param == null) { + param = speciesMovementParamMapper.selectDefault(); + } + if (param == null) { + throw new IllegalStateException("camera_trap_species_param is missing the __DEFAULT__ row"); + } + return SimuCamtrap.simuCamtrap(data, species, 50.0, null, 5000, 2.0, + param.getStepL(), param.getBias(), param.getRangeKm2() * 1000.0, ind, iteration); + } + + /** 判断分组内任一记录的学名或中文名命中过滤集合(忽略大小写)。 */ + private static boolean matchSpecies(List group, Set filter) { + for (CameraPoint p : group) { + if (p.getSpecies() != null && filter.contains(p.getSpecies().trim())) { + return true; + } + if (filter.stream().anyMatch(f -> p.getSpeciesName() != null + && f.equalsIgnoreCase(p.getSpeciesName().trim()))) { + return true; + } + if (p.getSpecies() != null && filter.stream().anyMatch(f -> f.equalsIgnoreCase(p.getSpecies().trim()))) { + return true; + } + } + return false; + } + + // ============================== densityEstimate ============================== + + @Override + public RasterResult densityEstimate(List data, double cellSize, int nTree, long seed) { + return DensityEstimate.densityEstimate(data, cellSize, nTree, seed); + } + + // ============================== habitatArea ============================== + + @Override + public HabitatAreaResult habitatArea(RasterResult raster, List presence, double manual) { + return HabitatAnalysis.habitatArea(raster, presence, manual); + } + + // ============================== habitatRange ============================== + + @Override + public HabitatRangeResult habitatRange(HabitatAreaResult area) { + return HabitatRange.habitatRange(area); + } + + // ============================== plotFootprint ============================== + + @Override + public FootprintTrace plotFootprint(List chain) { + return PlotFootprint.plotFootprint(chain); + } + + // ============================== moveBias ============================== + + @Override + public MoveBiasResult moveBias(List chain, int scale) { + return MoveBias.moveBias(chain, scale); + } + + // ============================== Rowcliffe ============================== + + @Override + public double[] rowcliffe(List data, double r, double theta, double v, double duration) { + return Rowcliffe.rowcliffe(data, r, theta, v, duration); + } + + // ============================== homeRange(s) ============================== + + @Override + public double homeRange(List chain) { + return HomeRange.homeRange(chain); + } + + @Override + public List homeRanges(List records) { + return HomeRange.homeRanges(records); + } + + // ============================== chainLength(s) ============================== + + @Override + public double chainLength(List chain) { + return ChainLength.chainLength(chain); + } + + @Override + public List chainLengths(List records) { + return ChainLength.chainLengths(records); + } + + // ===================================================================== + // DTO 转换辅助:JSONArray → List / ChainRecord / FootprintPoint + // ===================================================================== + + /** 前端传入 JSONArray(每项含 lon/lat/groupSize/date/time)→ CameraPoint 列表。 */ + public static List toCameraPoints(JSONArray arr) { + List list = new ArrayList<>(); + if (arr == null) { + return list; + } + for (int i = 0; i < arr.size(); i++) { + JSONObject o = arr.getJSONObject(i); + CameraPoint p = new CameraPoint(); + p.setLon(o.getDoubleValue("lon")); + p.setLat(o.getDoubleValue("lat")); + p.setGroupSize(o.getIntValue("groupSize")); + p.setDate(o.getString("date")); + p.setTime(o.getString("time")); + String sp = o.getString("species"); + p.setSpecies(sp == null ? "" : sp); + String spName = o.getString("speciesName"); + p.setSpeciesName(spName == null ? "" : spName); + list.add(p); + } + return list; + } + + public static List toFootprintPoints(JSONArray arr) { + List list = new ArrayList<>(); + if (arr == null) { + return list; + } + for (int i = 0; i < arr.size(); i++) { + JSONObject o = arr.getJSONObject(i); + FootprintPoint p = new FootprintPoint(); + p.setLon(o.getDoubleValue("lon")); + p.setLat(o.getDoubleValue("lat")); + p.setDate(o.getString("date")); + list.add(p); + } + return list; + } + + public static List toChainRecords(JSONArray arr) { + List list = new ArrayList<>(); + if (arr == null) { + return list; + } + for (int i = 0; i < arr.size(); i++) { + JSONObject o = arr.getJSONObject(i); + ChainRecord r = new ChainRecord(); + r.setSpecies(o.getString("species")); + r.setLine(o.getIntValue("line")); + r.setLon(o.getDoubleValue("lon")); + r.setLat(o.getDoubleValue("lat")); + r.setDate(o.getString("date")); + list.add(r); + } + return list; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/StatEsDataServiceImpl.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/StatEsDataServiceImpl.java new file mode 100644 index 0000000..a8d7695 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/service/impl/StatEsDataServiceImpl.java @@ -0,0 +1,296 @@ +package net.hzvtc.stat.service.impl; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.hzvtc.device.domain.DeviceInfo; +import net.hzvtc.device.service.DeviceInfoService; +import net.hzvtc.service.impl.CommonEsUtils; +import net.hzvtc.stat.dto.CameraPoint; +import net.hzvtc.stat.dto.ChainRecord; +import net.hzvtc.stat.dto.FootprintPoint; +import net.hzvtc.stat.service.StatEsDataService; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * {@link StatEsDataService} 实现:从 ES 索引 animal_valid_record + * 全量加载有效记录(search_after 分页),关联 device_info 表补齐经纬度。 + * + *

字段映射: + *

    + *
  • lon / lat ← device_info.longitude / latitude(按 device_id 关联,无坐标的记录跳过)
  • + *
  • groupSize ← count(缺失按 1)
  • + *
  • date / time ← record_time(转为 Asia/Shanghai 时区)
  • + *
  • species ← scientific_name(缺失用 species_name)
  • + *
  • line ← device_id(同一设备的同物种记录序列视为一条足迹链)
  • + *

+ */ +@Slf4j +@Service +@RequiredArgsConstructor +public class StatEsDataServiceImpl implements StatEsDataService { + + private static final String INDEX_NAME = "animal_valid_record"; + + private static final int PAGE_SIZE = 1000; + + /** 全量拉取上限,防止误查拖垮服务。 */ + private static final int MAX_ROWS = 200000; + + private static final DateTimeFormatter FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private static final ZoneId ZONE = ZoneId.of("Asia/Shanghai"); + + private final ElasticsearchClient elasticsearchClient; + + private final DeviceInfoService deviceInfoService; + + // ============================== 对外加载方法 ============================== + + @Override + public List loadCameraPoints(JSONObject param) { + List rows = searchAll(param); + Map locations = deviceLocations(rows); + List list = new ArrayList<>(rows.size()); + for (ObjectNode row : rows) { + double[] xy = locations.get(deviceId(row)); + LocalDateTime time = recordTime(row); + if (xy == null || time == null) { + continue; + } + CameraPoint p = new CameraPoint(); + p.setLon(xy[0]); + p.setLat(xy[1]); + p.setGroupSize(intVal(row, "count", 1)); + p.setDate(time.toLocalDate().toString()); + p.setTime(time.format(DateTimeFormatter.ofPattern("HH:mm:ss"))); + p.setSpecies(species(row)); + p.setSpeciesName(speciesName(row)); + list.add(p); + } + return list; + } + + @Override + public List loadFootprintPoints(JSONObject param) { + List rows = searchAll(param); + Map locations = deviceLocations(rows); + List list = new ArrayList<>(rows.size()); + for (ObjectNode row : rows) { + double[] xy = locations.get(deviceId(row)); + LocalDateTime time = recordTime(row); + if (xy == null || time == null) { + continue; + } + FootprintPoint p = new FootprintPoint(); + p.setLon(xy[0]); + p.setLat(xy[1]); + p.setDate(time.format(FMT)); + list.add(p); + } + list.sort(Comparator.comparing(FootprintPoint::getDate)); + return list; + } + + @Override + public List loadChainRecords(JSONObject param) { + List rows = searchAll(param); + Map locations = deviceLocations(rows); + List list = new ArrayList<>(rows.size()); + for (ObjectNode row : rows) { + double[] xy = locations.get(deviceId(row)); + LocalDateTime time = recordTime(row); + if (xy == null || time == null) { + continue; + } + ChainRecord r = new ChainRecord(); + r.setSpecies(species(row)); + Long dev = deviceId(row); + r.setLine(dev == null ? 0 : dev.intValue()); + r.setLon(xy[0]); + r.setLat(xy[1]); + r.setDate(time.format(FMT)); + list.add(r); + } + list.sort(Comparator.comparing(ChainRecord::getSpecies) + .thenComparing(ChainRecord::getLine) + .thenComparing(ChainRecord::getDate)); + return list; + } + + // ============================== ES 全量查询(search_after 分页) ============================== + + private List searchAll(JSONObject param) { + try { + BoolQuery bool = CommonEsUtils.createQuery(param).build(); + List rows = new ArrayList<>(); + List searchAfter = null; + while (rows.size() < MAX_ROWS) { + SearchRequest.Builder rb = new SearchRequest.Builder() + .index(INDEX_NAME) + .size(PAGE_SIZE) + .query(q -> q.bool(bool)) + .sort(s -> s.field(f -> f.field("id").order(SortOrder.Asc))); + if (searchAfter != null) { + rb.searchAfter(searchAfter); + } + SearchResponse resp = elasticsearchClient.search(rb.build(), ObjectNode.class); + List> hits = resp.hits().hits(); + if (hits.isEmpty()) { + break; + } + for (Hit h : hits) { + if (h.source() != null) { + rows.add(h.source()); + } + } + if (hits.size() < PAGE_SIZE) { + break; + } + searchAfter = hits.get(hits.size() - 1).sort(); + } + if (rows.size() >= MAX_ROWS) { + log.warn("stat ES 查询命中数达到上限 {},结果可能被截断", MAX_ROWS); + } + return rows; + } catch (IOException e) { + throw new RuntimeException("查询 ES 索引 " + INDEX_NAME + " 失败", e); + } + } + + // ============================== 设备坐标关联 ============================== + + /** 青海省经纬度范围(含少量边界余量): lon 89.4~103.2, lat 31.4~39.4。 */ + private static final double QINGHAI_LON_MIN = 89.4; + private static final double QINGHAI_LON_MAX = 103.2; + private static final double QINGHAI_LAT_MIN = 31.4; + private static final double QINGHAI_LAT_MAX = 39.4; + + /** 汇总记录中出现的 device_id,逐个查询(本次请求内缓存)取设备坐标。 */ + private Map deviceLocations(List rows) { + Set ids = new HashSet<>(); + for (ObjectNode row : rows) { + Long id = deviceId(row); + if (id != null) { + ids.add(id); + } + } + Map map = new HashMap<>(ids.size() * 2); + for (Long id : ids) { + try { + DeviceInfo device = deviceInfoService.selectDeviceInfoById(id); + if (device != null && inQinghai(device.getLongitude(), device.getLatitude())) { + map.put(id, new double[]{device.getLongitude(), device.getLatitude()}); + } else { + log.debug("设备 {} 不存在或坐标({} , {})不在青海范围内,相关记录被跳过", + id, device == null ? null : device.getLongitude(), + device == null ? null : device.getLatitude()); + } + } catch (Exception e) { + log.warn("查询设备 {} 坐标失败:{}", id, e.getMessage()); + } + } + return map; + } + + /** 经纬度非 0 且落在青海省范围内。 */ + private static boolean inQinghai(double lon, double lat) { + return lon > 0 && lat > 0 + && lon >= QINGHAI_LON_MIN && lon <= QINGHAI_LON_MAX + && lat >= QINGHAI_LAT_MIN && lat <= QINGHAI_LAT_MAX; + } + + // ============================== 字段解析辅助 ============================== + + private static Long deviceId(ObjectNode row) { + JsonNode n = row.get("device_id"); + return n == null || n.isNull() ? null : n.asLong(); + } + + private static int intVal(ObjectNode row, String field, int def) { + JsonNode n = row.get(field); + return n == null || n.isNull() ? def : n.asInt(def); + } + + private static String species(ObjectNode row) { + JsonNode n = row.get("scientific_name"); + if (n != null && !n.isNull() && !n.asText().trim().isEmpty()) { + return n.asText(); + } + n = row.get("species_name"); + return n == null || n.isNull() ? "" : n.asText(); + } + + /** 物种中文名(species_name),用于物种过滤时与学名共同匹配。 */ + private static String speciesName(ObjectNode row) { + JsonNode n = row.get("species_name"); + return n == null || n.isNull() ? "" : n.asText().trim(); + } + + /** + * 解析 record_time(ES date 类型,_source 中可能为 + * "yyyy-MM-dd HH:mm:ss" / ISO8601 带时区 / epoch 毫秒), + * 统一转为 Asia/Shanghai 本地时间。 + */ + private static LocalDateTime recordTime(ObjectNode row) { + JsonNode n = row.get("record_time"); + if (n == null || n.isNull()) { + return null; + } + if (n.isNumber()) { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(n.longValue()), ZONE); + } + String s = n.asText().trim(); + if (s.isEmpty()) { + return null; + } + // 带时区(Z / +08:00)→ 转本地时区 + if (s.endsWith("Z") || s.matches(".*[+-]\\d{2}:?\\d{2}$")) { + try { + return OffsetDateTime.parse(s).atZoneSameInstant(ZONE).toLocalDateTime(); + } catch (Exception ignore) { + // 落到通用解析 + } + } + // 无时区,按本地时间解析 + String t = s.replace('T', ' '); + try { + if (t.length() >= 19) { + return LocalDateTime.parse(t.substring(0, 19), FMT); + } + if (t.length() == 16) { + return LocalDateTime.parse(t + ":00", FMT); + } + if (t.length() == 10) { + return LocalDate.parse(t).atStartOfDay(); + } + } catch (Exception ignore) { + // 未知格式 + } + return null; + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/DensityUtils.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/DensityUtils.java new file mode 100644 index 0000000..cd66f8d --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/DensityUtils.java @@ -0,0 +1,60 @@ +package net.hzvtc.stat.util; + +import net.hzvtc.stat.dto.DensitySeries; + +/** + * 核密度估计(Gaussian KDE)。 + * + *

cameratrapR 中 {@code density(times, bw)} 调用的就是 R stats::density(默认高斯核)。 + * 本类按相同语义实现:在等距网格上对样本做高斯核密度估计,返回 (x[], y[])。

+ * + *

核密度公式: + *

+ *   f(x) = (1 / (N * bw * sqrt(2π))) * Σ exp(-0.5 * ((x - xi) / bw)^2)
+ * 
+ *

+ */ +public final class DensityUtils { + + private DensityUtils() { + } + + private static final double SQRT_2PI = Math.sqrt(2.0 * Math.PI); + + /** + * 在 [xmin, xmax] 范围内等距取 n 个点,估计样本的高斯核密度。 + * + * @param samples 原始样本(一维数组) + * @param bw 带宽(与 R 中 bw 含义一致;dailyRhythm 使用 bw=0.3 与 bw=2) + * @param n 输出采样点数(默认 512 与 R 一致) + * @param xmin x 轴下限 + * @param xmax x 轴上限 + * @param label 曲线标签(写入结果) + * @return {@link DensitySeries} + */ + public static DensitySeries gaussianKde(double[] samples, double bw, int n, double xmin, double xmax, String label) { + if (samples.length == 0 || bw <= 0 || n < 2) { + return new DensitySeries(new double[0], new double[0], bw, label); + } + + double[] x = new double[n]; + double step = (xmax - xmin) / (n - 1); + for (int i = 0; i < n; i++) { + x[i] = xmin + step * i; + } + + double[] y = new double[n]; + double norm = 1.0 / (samples.length * bw * SQRT_2PI); + for (int i = 0; i < n; i++) { + double xi = x[i]; + double acc = 0.0; + for (double s : samples) { + double z = (xi - s) / bw; + acc += Math.exp(-0.5 * z * z); + } + y[i] = acc * norm; + } + + return new DensitySeries(x, y, bw, label); + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/GeoUtils.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/GeoUtils.java new file mode 100644 index 0000000..d7e7e06 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/GeoUtils.java @@ -0,0 +1,130 @@ +package net.hzvtc.stat.util; + +import java.util.List; + +/** + * 地理坐标工具:WGS84 经纬度 → UTM 米。 + * + *

cameratrapR 的 geo2proj 默认使用 WGS84 → UTM(按数据经度均值自动推算 zone)。 + * 本类实现相同的转换,使用 WGS84 椭球参数与 Transverse Mercator(Kruger)公式。

+ * + *

UTM zone 推算(与 R 代码一致):zone = floor((mean(lon)+180)/6) %% 60 + 1。

+ */ +public final class GeoUtils { + + private GeoUtils() { + } + + /** WGS84 长半轴(m)。 */ + private static final double A = 6378137.0; + + /** WGS84 第一偏心率平方。 */ + private static final double E2 = 0.00669437999014; + + /** WGS84 第二偏心率平方。 */ + private static final double EP2 = E2 / (1.0 - E2); + + /** UTM 比例因子。 */ + private static final double K0 = 0.9996; + + /** + * 将经纬度转换为 UTM 米。 + * + * @param lonDeg 经度(度) + * @param latDeg 纬度(度) + * @param zone UTM 区号(1..60) + * @return double[2] = {easting(m), northing(m)},东向已加 500000 偏移。 + */ + public static double[] wgs84ToUtm(double lonDeg, double latDeg, int zone) { + double lonRad = Math.toRadians(lonDeg); + double latRad = Math.toRadians(latDeg); + + // 中央经线 + double lon0Rad = Math.toRadians((zone - 1) * 6.0 - 180.0 + 3.0); + + double sinLat = Math.sin(latRad); + double cosLat = Math.cos(latRad); + double tanLat = Math.tan(latRad); + + double n = A / Math.sqrt(1.0 - E2 * sinLat * sinLat); + double t = tanLat * tanLat; + double c = EP2 * cosLat * cosLat; + double a = cosLat * (lonRad - lon0Rad); + + // 子午线弧长 M + double m = A * ((1.0 - E2 / 4.0 - 3.0 * E2 * E2 / 64.0 - 5.0 * E2 * E2 * E2 / 256.0) * latRad + - (3.0 * E2 / 8.0 + 3.0 * E2 * E2 / 32.0 + 45.0 * E2 * E2 * E2 / 1024.0) * Math.sin(2.0 * latRad) + + (15.0 * E2 * E2 / 256.0 + 45.0 * E2 * E2 * E2 / 1024.0) * Math.sin(4.0 * latRad) + - (35.0 * E2 * E2 * E2 / 3072.0) * Math.sin(6.0 * latRad)); + + double easting = K0 * n * (a + (1.0 - t + c) * a * a * a / 6.0 + + (5.0 - 18.0 * t + t * t + 72.0 * c - 58.0 * EP2) * Math.pow(a, 5) / 120.0) + + 500000.0; + + double northing = K0 * (m + n * tanLat * (a * a / 2.0 + + (5.0 - t + 9.0 * c + 4.0 * c * c) * Math.pow(a, 4) / 24.0 + + (61.0 - 58.0 * t + t * t + 600.0 * c - 330.0 * EP2) * Math.pow(a, 6) / 720.0)); + + // 南半球偏移 10 000 000 + if (latDeg < 0) { + northing += 10000000.0; + } + + return new double[]{easting, northing}; + } + + /** + * 自动按数据经度均值推算 UTM zone,转换所有点为米。 + * + * @param lons 经度数组(度) + * @param lats 纬度数组(度) + * @return double[n][2] = {easting(m), northing(m)} + */ + public static double[][] wgs84ToUtmAuto(double[] lons, double[] lats) { + if (lons.length != lats.length) { + throw new IllegalArgumentException("lons and lats length mismatch"); + } + double meanLon = 0.0; + for (double lon : lons) { + meanLon += lon; + } + meanLon /= lons.length; + + int zone = (int) Math.floor((meanLon + 180.0) / 6.0) % 60 + 1; + if (zone < 1) { + zone = 1; + } + + double[][] out = new double[lons.length][2]; + for (int i = 0; i < lons.length; i++) { + out[i] = wgs84ToUtm(lons[i], lats[i], zone); + } + return out; + } + + /** + * 自动 zone 并转换带 (lon, lat) 字段的对象列表。 + */ + public static double[][] wgs84ToUtmAuto(List points, LonGetter lonGetter, LatGetter latGetter) { + double[] lons = new double[points.size()]; + double[] lats = new double[points.size()]; + for (int i = 0; i < points.size(); i++) { + T p = points.get(i); + lons[i] = lonGetter.getLon(p); + lats[i] = latGetter.getLat(p); + } + return wgs84ToUtmAuto(lons, lats); + } + + /** 取对象的经度。 */ + @FunctionalInterface + public interface LonGetter { + double getLon(T obj); + } + + /** 取对象的纬度。 */ + @FunctionalInterface + public interface LatGetter { + double getLat(T obj); + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/RandomUtils.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/RandomUtils.java new file mode 100644 index 0000000..d0093a8 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/RandomUtils.java @@ -0,0 +1,83 @@ +package net.hzvtc.stat.util; + +import java.util.Random; + +/** + * 随机数工具:均匀、高斯(与 R runif / rnorm 语义一致)。 + * + *

默认使用固定种子(等价 R 的 set.seed(1234)), + * 保证同样输入下结果可复现;可在每次查询前调用 {@link #setSeed(long)} 重置。

+ */ +public final class RandomUtils { + + private RandomUtils() { + } + + /** 默认种子(与接口层 seed 默认值一致)。 */ + public static final long DEFAULT_SEED = 1234L; + + /** 每线程独立的种子化随机源(ThreadLocalRandom 无法指定种子,故用 ThreadLocal<Random>)。 */ + private static final ThreadLocal RNG = ThreadLocal.withInitial(() -> new Random(DEFAULT_SEED)); + + /** + * 重置随机种子(等价 R 的 set.seed)。 + * 供每次查询/每个物种模拟前调用,保证同样条件下结果可复现。 + */ + public static void setSeed(long seed) { + RNG.set(new Random(seed)); + } + + /** R 的 runif(1, min, max) 等价物。 */ + public static double uniform(double min, double max) { + return RNG.get().nextDouble() * (max - min) + min; + } + + /** + * R 的 rnorm(1, mean, sd) 等价物(Box-Muller 变换,基于同一种子源)。 + */ + public static double gaussian(double mean, double sd) { + return gauss() * sd + mean; + } + + /** 一次性生成 n 个高斯样本。 */ + public static double[] gaussianSamples(double mean, double sd, int n) { + double[] out = new double[n]; + for (int i = 0; i < n; i++) { + out[i] = gaussian(mean, sd); + } + return out; + } + + /** + * 离散采样(按概率权重抽取一个 index)。等价 R 的 sample(x, 1, prob=p)。 + * + * @param weights 非负权重数组 + * @return 命中的下标 + */ + public static int weightedSample(double[] weights) { + double sum = 0.0; + for (double w : weights) { + sum += w; + } + double r = uniform(0.0, sum); + double acc = 0.0; + for (int i = 0; i < weights.length; i++) { + acc += weights[i]; + if (r <= acc) { + return i; + } + } + return weights.length - 1; + } + + /** Box-Muller 标准正态样本。 */ + private static double gauss() { + Random r = RNG.get(); + double u1; + do { + u1 = r.nextDouble(); + } while (u1 <= 0.0); + double u2 = r.nextDouble(); + return Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2); + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/StatisticsUtils.java b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/StatisticsUtils.java new file mode 100644 index 0000000..2f7b961 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/stat/util/StatisticsUtils.java @@ -0,0 +1,60 @@ +package net.hzvtc.stat.util; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; + +/** + * 描述统计工具:mean / sd / quantile(与 R mean / sd / quantile 语义一致)。 + * + *

使用 commons-math3 的 {@link DescriptiveStatistics},分位数采用 R type-7(即线性插值)。

+ */ +public final class StatisticsUtils { + + private StatisticsUtils() { + } + + public static double mean(double[] values) { + DescriptiveStatistics s = new DescriptiveStatistics(values); + return s.getMean(); + } + + public static double sd(double[] values) { + if (values.length < 2) { + return 0.0; + } + return Math.sqrt(new DescriptiveStatistics(values).getVariance()); + } + + /** + * R type-7 分位数。 + */ + public static double quantile(double[] values, double p) { + DescriptiveStatistics s = new DescriptiveStatistics(values); + return s.getPercentile(p * 100.0); + } + + public static double max(double[] values) { + double m = Double.NEGATIVE_INFINITY; + for (double v : values) { + if (v > m) { + m = v; + } + } + return m; + } + + public static double min(double[] values) { + double m = Double.POSITIVE_INFINITY; + for (double v : values) { + if (v < m) { + m = v; + } + } + return m; + } + + public static double[] sort(double[] values) { + double[] copy = values.clone(); + java.util.Arrays.sort(copy); + return copy; + } +} \ No newline at end of file diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeDeserializer.java b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeDeserializer.java new file mode 100644 index 0000000..c98282b --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeDeserializer.java @@ -0,0 +1,22 @@ +package net.hzvtc.sys.config; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +public class CustomDateTimeDeserializer extends JsonDeserializer { + + private static final DateTimeFormatter INPUT_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + + @Override + public ZonedDateTime deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + String dateStr = p.getText(); + return ZonedDateTime.parse(dateStr, INPUT_FORMATTER); + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeSerializer.java b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeSerializer.java new file mode 100644 index 0000000..c6f5bdb --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/CustomDateTimeSerializer.java @@ -0,0 +1,21 @@ +package net.hzvtc.sys.config; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +public class CustomDateTimeSerializer extends JsonSerializer { + + private static final DateTimeFormatter OUTPUT_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Override + public void serialize(ZonedDateTime value, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + String formattedDate = value.format(OUTPUT_FORMATTER); + gen.writeString(formattedDate); + } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/ESClientConfig.java b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/ESClientConfig.java new file mode 100644 index 0000000..29fbaac --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/ESClientConfig.java @@ -0,0 +1,168 @@ +package net.hzvtc.sys.config; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ClassPathResource; +import org.springframework.util.StringUtils; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; + +@Setter +@Configuration +@Slf4j +public class ESClientConfig { + + @Value("${spring.elastic.hosts}") + private String hosts; + +// @Setter +// @Value("${spring.elastic.port}") +// private int port; + + @Value("${spring.elastic.http}") + private String http; + + @Value("${spring.elastic.apiKey}") + private String apiKey; + + @Value("${spring.elastic.username}") + private String username; + + @Value("${spring.elastic.password}") + private String passwd; + + + private static String certName; + + + @Value("${spring.elastic.certName}") + public void setAppKey(String certName) { + ESClientConfig.certName = certName; + } + + + private HttpHost[] toHttpHost() { + if (!StringUtils.hasLength(hosts)) { + throw new RuntimeException("invalid elasticsearch configuration"); + } + + String[] hostArray = hosts.split(","); + HttpHost[] httpHosts = new HttpHost[hostArray.length]; + HttpHost httpHost; + for (int i = 0; i < hostArray.length; i++) { + String[] strings = hostArray[i].split(":"); + httpHost = new HttpHost(strings[0], Integer.parseInt(strings[1]), "https"); + httpHosts[i] = httpHost; + } + + return httpHosts; + } + + @Bean("clientByPasswd") + public ElasticsearchClient clientByPasswd() { + ElasticsearchTransport transport = getElasticsearchTransport(username, passwd, toHttpHost()); + return new ElasticsearchClient(transport); + } + + private static SSLContext buildSSLContext() { +// ClassPathResource resource = new ClassPathResource("es_ca.crt"); + ClassPathResource resource = new ClassPathResource(certName); + SSLContext sslContext = null; + try { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + Certificate trustedCa; + try (InputStream is = resource.getInputStream()) { + trustedCa = factory.generateCertificate(is); + } + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null, null); + trustStore.setCertificateEntry("ca", trustedCa); + SSLContextBuilder sslContextBuilder = SSLContexts.custom() + .loadTrustMaterial(trustStore, null); + sslContext = sslContextBuilder.build(); + } catch (IOException | KeyStoreException | NoSuchAlgorithmException | KeyManagementException | + CertificateException e) { + log.error("ES连接认证失败", e); + } + + return sslContext; + } + + private static ElasticsearchTransport getElasticsearchTransport(String username, String passwd, HttpHost... hosts) { + // 账号密码的配置 + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, passwd)); + + // 自签证书的设置,并且还包含了账号密码 + RestClientBuilder.HttpClientConfigCallback callback = httpAsyncClientBuilder -> httpAsyncClientBuilder + .setSSLContext(buildSSLContext()) + .setMaxConnTotal(100) + .setMaxConnPerRoute(100) + .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setDefaultCredentialsProvider(credentialsProvider); + + // 用builder创建RestClient对象 + RestClient client = RestClient + .builder(hosts) + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(5000).setSocketTimeout(120000)) + .setHttpClientConfigCallback(callback) + .build(); + + return new RestClientTransport(client, new JacksonJsonpMapper(JacksonConfiguration.createObjectMapper())); + } + + private static ElasticsearchTransport getElasticsearchTransport(String apiKey, HttpHost... hosts) { + // 将ApiKey放入header中 + Header[] headers = new Header[]{new BasicHeader("Authorization", "ApiKey " + apiKey)}; + + // es自签证书的设置 + RestClientBuilder.HttpClientConfigCallback callback = httpAsyncClientBuilder -> httpAsyncClientBuilder + .setSSLContext(buildSSLContext()) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + + // 用builder创建RestClient对象 + RestClient client = RestClient + .builder(hosts) + .setHttpClientConfigCallback(callback) + .setDefaultHeaders(headers) + .build(); + + return new RestClientTransport(client, new JacksonJsonpMapper()); + } + +// @Bean("clientByApiKey") +// public ElasticsearchClient clientByApiKey() throws Exception { +// ElasticsearchTransport transport = getElasticsearchTransport(apiKey, toHttpHost()); +// return new ElasticsearchClient(transport); +// } +} diff --git a/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/JacksonConfiguration.java b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/JacksonConfiguration.java new file mode 100644 index 0000000..92b32b7 --- /dev/null +++ b/ai-xuebao-es/src/main/java/net/hzvtc/sys/config/JacksonConfiguration.java @@ -0,0 +1,19 @@ +package net.hzvtc.sys.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import java.time.ZonedDateTime; + +public class JacksonConfiguration { + public static ObjectMapper createObjectMapper() { + return new ObjectMapper() + .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .registerModule(new JavaTimeModule() + .addSerializer(ZonedDateTime.class, new CustomDateTimeSerializer()) + .addDeserializer(ZonedDateTime.class, new CustomDateTimeDeserializer())); + } +} diff --git a/ai-xuebao-es/src/main/resources/mapper/EsResultRecordDetailsMapper.xml b/ai-xuebao-es/src/main/resources/mapper/EsResultRecordDetailsMapper.xml new file mode 100644 index 0000000..bc57d03 --- /dev/null +++ b/ai-xuebao-es/src/main/resources/mapper/EsResultRecordDetailsMapper.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataScopeAspect.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataScopeAspect.java new file mode 100644 index 0000000..dfe312d --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataScopeAspect.java @@ -0,0 +1,149 @@ +package net.hzvtc.framework.aspectj; + +import net.hzvtc.common.core.domain.BaseEntity; +import net.hzvtc.common.core.domain.entity.SysRole; +import net.hzvtc.common.core.domain.entity.SysUser; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.StringUtils; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.springframework.stereotype.Component; +import net.hzvtc.common.annotation.DataScope; + +/** + * 数据过滤处理 + * + * @author admin + */ +@Aspect +@Component +public class DataScopeAspect +{ + /** + * 全部数据权限 + */ + public static final String DATA_SCOPE_ALL = "1"; + + /** + * 自定数据权限 + */ + public static final String DATA_SCOPE_CUSTOM = "2"; + + /** + * 部门数据权限 + */ + public static final String DATA_SCOPE_DEPT = "3"; + + /** + * 部门及以下数据权限 + */ + public static final String DATA_SCOPE_DEPT_AND_CHILD = "4"; + + /** + * 仅本人数据权限 + */ + public static final String DATA_SCOPE_SELF = "5"; + + /** + * 数据权限过滤关键字 + */ + public static final String DATA_SCOPE = "dataScope"; + + @Before("@annotation(controllerDataScope)") + public void doBefore(JoinPoint point, DataScope controllerDataScope) throws Throwable + { + clearDataScope(point); + handleDataScope(point, controllerDataScope); + } + + protected void handleDataScope(final JoinPoint joinPoint, DataScope controllerDataScope) + { + // 获取当前的用户 + LoginUser loginUser = SecurityUtils.getLoginUser(); + if (StringUtils.isNotNull(loginUser)) + { + SysUser currentUser = loginUser.getUser(); + // 如果是超级管理员,则不过滤数据 + if (StringUtils.isNotNull(currentUser) && !currentUser.isAdmin()) + { + dataScopeFilter(joinPoint, currentUser, controllerDataScope.deptAlias(), + controllerDataScope.userAlias()); + } + } + } + + /** + * 数据范围过滤 + * + * @param joinPoint 切点 + * @param user 用户 + * @param userAlias 别名 + */ + public static void dataScopeFilter(JoinPoint joinPoint, SysUser user, String deptAlias, String userAlias) + { + StringBuilder sqlString = new StringBuilder(); + + for (SysRole role : user.getRoles()) + { + String dataScope = role.getDataScope(); + if (DATA_SCOPE_ALL.equals(dataScope)) + { + sqlString = new StringBuilder(); + break; + } + else if (DATA_SCOPE_CUSTOM.equals(dataScope)) + { + sqlString.append(StringUtils.format( + " OR {}.dept_id IN ( SELECT dept_id FROM sys_role_dept WHERE role_id = {} ) ", deptAlias, + role.getRoleId())); + } + else if (DATA_SCOPE_DEPT.equals(dataScope)) + { + sqlString.append(StringUtils.format(" OR {}.dept_id = {} ", deptAlias, user.getDeptId())); + } + else if (DATA_SCOPE_DEPT_AND_CHILD.equals(dataScope)) + { + sqlString.append(StringUtils.format( + " OR {}.dept_id IN ( SELECT dept_id FROM sys_dept WHERE dept_id = {} or find_in_set( {} , ancestors ) )", + deptAlias, user.getDeptId(), user.getDeptId())); + } + else if (DATA_SCOPE_SELF.equals(dataScope)) + { + if (StringUtils.isNotBlank(userAlias)) + { + sqlString.append(StringUtils.format(" OR {}.user_id = {} ", userAlias, user.getUserId())); + } + else + { + // 数据权限为仅本人且没有userAlias别名不查询任何数据 + sqlString.append(" OR 1=0 "); + } + } + } + + if (StringUtils.isNotBlank(sqlString.toString())) + { + Object params = joinPoint.getArgs()[0]; + if (StringUtils.isNotNull(params) && params instanceof BaseEntity) + { + BaseEntity baseEntity = (BaseEntity) params; + baseEntity.getParams().put(DATA_SCOPE, " AND (" + sqlString.substring(4) + ")"); + } + } + } + + /** + * 拼接权限sql前先清空params.dataScope参数防止注入 + */ + private void clearDataScope(final JoinPoint joinPoint) + { + Object params = joinPoint.getArgs()[0]; + if (StringUtils.isNotNull(params) && params instanceof BaseEntity) + { + BaseEntity baseEntity = (BaseEntity) params; + baseEntity.getParams().put(DATA_SCOPE, ""); + } + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataSourceAspect.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataSourceAspect.java new file mode 100644 index 0000000..eb1ba0d --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/DataSourceAspect.java @@ -0,0 +1,73 @@ +package net.hzvtc.framework.aspectj; + +import java.util.Objects; + +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.framework.datasource.DynamicDataSourceContextHolder; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import net.hzvtc.common.annotation.DataSource; + +/** + * 多数据源处理 + * + * @author admin + */ +@Aspect +@Order(1) +@Component +public class DataSourceAspect +{ + protected Logger logger = LoggerFactory.getLogger(getClass()); + + @Pointcut("@annotation(net.hzvtc.common.annotation.DataSource)" + + "|| @within(net.hzvtc.common.annotation.DataSource)") + public void dsPointCut() + { + + } + + @Around("dsPointCut()") + public Object around(ProceedingJoinPoint point) throws Throwable + { + DataSource dataSource = getDataSource(point); + + if (StringUtils.isNotNull(dataSource)) + { + DynamicDataSourceContextHolder.setDataSourceType(dataSource.value().name()); + } + + try + { + return point.proceed(); + } + finally + { + // 销毁数据源 在执行方法之后 + DynamicDataSourceContextHolder.clearDataSourceType(); + } + } + + /** + * 获取需要切换的数据源 + */ + public DataSource getDataSource(ProceedingJoinPoint point) + { + MethodSignature signature = (MethodSignature) point.getSignature(); + DataSource dataSource = AnnotationUtils.findAnnotation(signature.getMethod(), DataSource.class); + if (Objects.nonNull(dataSource)) + { + return dataSource; + } + + return AnnotationUtils.findAnnotation(signature.getDeclaringType(), DataSource.class); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/LogAspect.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/LogAspect.java new file mode 100644 index 0000000..c52487a --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/LogAspect.java @@ -0,0 +1,219 @@ +package net.hzvtc.framework.aspectj; + +import java.util.Collection; +import java.util.Map; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.enums.BusinessStatus; +import net.hzvtc.common.enums.HttpMethod; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.common.utils.ip.IpUtils; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.AfterReturning; +import org.aspectj.lang.annotation.AfterThrowing; +import org.aspectj.lang.annotation.Aspect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.validation.BindingResult; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.HandlerMapping; +import com.alibaba.fastjson.JSON; +import net.hzvtc.common.annotation.Log; +import net.hzvtc.framework.manager.AsyncManager; +import net.hzvtc.framework.manager.factory.AsyncFactory; +import net.hzvtc.system.domain.SysOperLog; + +/** + * 操作日志记录处理 + * + * @author admin + */ +@Aspect +@Component +public class LogAspect +{ + private static final Logger log = LoggerFactory.getLogger(LogAspect.class); + + /** + * 处理完请求后执行 + * + * @param joinPoint 切点 + */ + @AfterReturning(pointcut = "@annotation(controllerLog)", returning = "jsonResult") + public void doAfterReturning(JoinPoint joinPoint, Log controllerLog, Object jsonResult) + { + handleLog(joinPoint, controllerLog, null, jsonResult); + } + + /** + * 拦截异常操作 + * + * @param joinPoint 切点 + * @param e 异常 + */ + @AfterThrowing(value = "@annotation(controllerLog)", throwing = "e") + public void doAfterThrowing(JoinPoint joinPoint, Log controllerLog, Exception e) + { + handleLog(joinPoint, controllerLog, e, null); + } + + protected void handleLog(final JoinPoint joinPoint, Log controllerLog, final Exception e, Object jsonResult) + { + try + { + + // 获取当前的用户 + LoginUser loginUser = SecurityUtils.getLoginUser(); + + // *========数据库日志=========*// + SysOperLog operLog = new SysOperLog(); + operLog.setStatus(BusinessStatus.SUCCESS.ordinal()); + // 请求的地址 + String ip = IpUtils.getIpAddr(ServletUtils.getRequest()); + operLog.setOperIp(ip); + operLog.setOperUrl(ServletUtils.getRequest().getRequestURI()); + if (loginUser != null) + { + operLog.setOperName(loginUser.getUsername()); + } + + if (e != null) + { + operLog.setStatus(BusinessStatus.FAIL.ordinal()); + operLog.setErrorMsg(StringUtils.substring(e.getMessage(), 0, 2000)); + } + // 设置方法名称 + String className = joinPoint.getTarget().getClass().getName(); + String methodName = joinPoint.getSignature().getName(); + operLog.setMethod(className + "." + methodName + "()"); + // 设置请求方式 + operLog.setRequestMethod(ServletUtils.getRequest().getMethod()); + // 处理设置注解上的参数 + getControllerMethodDescription(joinPoint, controllerLog, operLog, jsonResult); + // 保存数据库 + AsyncManager.me().execute(AsyncFactory.recordOper(operLog)); + } + catch (Exception exp) + { + // 记录本地异常日志 + log.error("==前置通知异常=="); + log.error("异常信息:{}", exp.getMessage()); + exp.printStackTrace(); + } + } + + /** + * 获取注解中对方法的描述信息 用于Controller层注解 + * + * @param log 日志 + * @param operLog 操作日志 + * @throws Exception + */ + public void getControllerMethodDescription(JoinPoint joinPoint, Log log, SysOperLog operLog, Object jsonResult) throws Exception + { + // 设置action动作 + operLog.setBusinessType(log.businessType().ordinal()); + // 设置标题 + operLog.setTitle(log.title()); + // 设置操作人类别 + operLog.setOperatorType(log.operatorType().ordinal()); + // 是否需要保存request,参数和值 + if (log.isSaveRequestData()) + { + // 获取参数的信息,传入到数据库中。 + setRequestValue(joinPoint, operLog); + } + // 是否需要保存response,参数和值 + if (log.isSaveResponseData() && StringUtils.isNotNull(jsonResult)) + { + operLog.setJsonResult(StringUtils.substring(JSON.toJSONString(jsonResult), 0, 2000)); + } + } + + /** + * 获取请求的参数,放到log中 + * + * @param operLog 操作日志 + * @throws Exception 异常 + */ + private void setRequestValue(JoinPoint joinPoint, SysOperLog operLog) throws Exception + { + String requestMethod = operLog.getRequestMethod(); + if (HttpMethod.PUT.name().equals(requestMethod) || HttpMethod.POST.name().equals(requestMethod)) + { + String params = argsArrayToString(joinPoint.getArgs()); + operLog.setOperParam(StringUtils.substring(params, 0, 2000)); + } + else + { + Map paramsMap = (Map) ServletUtils.getRequest().getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE); + operLog.setOperParam(StringUtils.substring(paramsMap.toString(), 0, 2000)); + } + } + + /** + * 参数拼装 + */ + private String argsArrayToString(Object[] paramsArray) + { + String params = ""; + if (paramsArray != null && paramsArray.length > 0) + { + for (Object o : paramsArray) + { + if (StringUtils.isNotNull(o) && !isFilterObject(o)) + { + try + { + Object jsonObj = JSON.toJSON(o); + params += jsonObj.toString() + " "; + } + catch (Exception e) + { + } + } + } + } + return params.trim(); + } + + /** + * 判断是否需要过滤的对象。 + * + * @param o 对象信息。 + * @return 如果是需要过滤的对象,则返回true;否则返回false。 + */ + @SuppressWarnings("rawtypes") + public boolean isFilterObject(final Object o) + { + Class clazz = o.getClass(); + if (clazz.isArray()) + { + return clazz.getComponentType().isAssignableFrom(MultipartFile.class); + } + else if (Collection.class.isAssignableFrom(clazz)) + { + Collection collection = (Collection) o; + for (Object value : collection) + { + return value instanceof MultipartFile; + } + } + else if (Map.class.isAssignableFrom(clazz)) + { + Map map = (Map) o; + for (Object value : map.entrySet()) + { + Map.Entry entry = (Map.Entry) value; + return entry.getValue() instanceof MultipartFile; + } + } + return o instanceof MultipartFile || o instanceof HttpServletRequest || o instanceof HttpServletResponse + || o instanceof BindingResult; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/RateLimiterAspect.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/RateLimiterAspect.java new file mode 100644 index 0000000..328e9dc --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/aspectj/RateLimiterAspect.java @@ -0,0 +1,92 @@ +package net.hzvtc.framework.aspectj; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import net.hzvtc.common.enums.LimitType; +import net.hzvtc.common.exception.ServiceException; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.common.utils.ip.IpUtils; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.stereotype.Component; +import net.hzvtc.common.annotation.RateLimiter; + +/** + * 限流处理 + * + * @author admin + */ +@Aspect +@Component +public class RateLimiterAspect +{ + private static final Logger log = LoggerFactory.getLogger(RateLimiterAspect.class); + + private RedisTemplate redisTemplate; + + private RedisScript limitScript; + + @Autowired + public void setRedisTemplate1(RedisTemplate redisTemplate) + { + this.redisTemplate = redisTemplate; + } + + @Autowired + public void setLimitScript(RedisScript limitScript) + { + this.limitScript = limitScript; + } + + @Before("@annotation(rateLimiter)") + public void doBefore(JoinPoint point, RateLimiter rateLimiter) throws Throwable + { + String key = rateLimiter.key(); + int time = rateLimiter.time(); + int count = rateLimiter.count(); + + String combineKey = getCombineKey(rateLimiter, point); + List keys = Collections.singletonList(combineKey); + try + { + Long number = redisTemplate.execute(limitScript, keys, count, time); + if (StringUtils.isNull(number) || number.intValue() > count) + { + throw new ServiceException("访问过于频繁,请稍候再试"); + } + log.info("限制请求'{}',当前请求'{}',缓存key'{}'", count, number.intValue(), key); + } + catch (ServiceException e) + { + throw e; + } + catch (Exception e) + { + throw new RuntimeException("服务器限流异常,请稍候再试"); + } + } + + public String getCombineKey(RateLimiter rateLimiter, JoinPoint point) + { + StringBuffer stringBuffer = new StringBuffer(rateLimiter.key()); + if (rateLimiter.limitType() == LimitType.IP) + { + stringBuffer.append(IpUtils.getIpAddr(ServletUtils.getRequest())).append("-"); + } + MethodSignature signature = (MethodSignature) point.getSignature(); + Method method = signature.getMethod(); + Class targetClass = method.getDeclaringClass(); + stringBuffer.append(targetClass.getName()).append("-").append(method.getName()); + return stringBuffer.toString(); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ApplicationConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ApplicationConfig.java new file mode 100644 index 0000000..442d3b8 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ApplicationConfig.java @@ -0,0 +1,30 @@ +package net.hzvtc.framework.config; + +import java.util.TimeZone; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; + +/** + * 程序注解配置 + * + * @author admin + */ +@Configuration +// 表示通过aop框架暴露该代理对象,AopContext能够访问 +@EnableAspectJAutoProxy(exposeProxy = true) +// 指定要扫描的Mapper类的包的路径 +@MapperScan("net.hzvtc.**.mapper") +public class ApplicationConfig +{ + /** + * 时区配置 + */ + @Bean + public Jackson2ObjectMapperBuilderCustomizer jacksonObjectMapperCustomization() + { + return jacksonObjectMapperBuilder -> jacksonObjectMapperBuilder.timeZone(TimeZone.getDefault()); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/CaptchaConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/CaptchaConfig.java new file mode 100644 index 0000000..369f21a --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/CaptchaConfig.java @@ -0,0 +1,83 @@ +package net.hzvtc.framework.config; + +import java.util.Properties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import com.google.code.kaptcha.impl.DefaultKaptcha; +import com.google.code.kaptcha.util.Config; +import static com.google.code.kaptcha.Constants.*; + +/** + * 验证码配置 + * + * @author admin + */ +@Configuration +public class CaptchaConfig +{ + @Bean(name = "captchaProducer") + public DefaultKaptcha getKaptchaBean() + { + DefaultKaptcha defaultKaptcha = new DefaultKaptcha(); + Properties properties = new Properties(); + // 是否有边框 默认为true 我们可以自己设置yes,no + properties.setProperty(KAPTCHA_BORDER, "yes"); + // 验证码文本字符颜色 默认为Color.BLACK + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_COLOR, "black"); + // 验证码图片宽度 默认为200 + properties.setProperty(KAPTCHA_IMAGE_WIDTH, "160"); + // 验证码图片高度 默认为50 + properties.setProperty(KAPTCHA_IMAGE_HEIGHT, "60"); + // 验证码文本字符大小 默认为40 + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_SIZE, "38"); + // KAPTCHA_SESSION_KEY + properties.setProperty(KAPTCHA_SESSION_CONFIG_KEY, "kaptchaCode"); + // 验证码文本字符长度 默认为5 + properties.setProperty(KAPTCHA_TEXTPRODUCER_CHAR_LENGTH, "4"); + // 验证码文本字体样式 默认为new Font("Arial", 1, fontSize), new Font("Courier", 1, fontSize) + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_NAMES, "Arial,Courier"); + // 图片样式 水纹com.google.code.kaptcha.impl.WaterRipple 鱼眼com.google.code.kaptcha.impl.FishEyeGimpy 阴影com.google.code.kaptcha.impl.ShadowGimpy + properties.setProperty(KAPTCHA_OBSCURIFICATOR_IMPL, "com.google.code.kaptcha.impl.ShadowGimpy"); + Config config = new Config(properties); + defaultKaptcha.setConfig(config); + return defaultKaptcha; + } + + @Bean(name = "captchaProducerMath") + public DefaultKaptcha getKaptchaBeanMath() + { + DefaultKaptcha defaultKaptcha = new DefaultKaptcha(); + Properties properties = new Properties(); + // 是否有边框 默认为true 我们可以自己设置yes,no + properties.setProperty(KAPTCHA_BORDER, "yes"); + // 边框颜色 默认为Color.BLACK + properties.setProperty(KAPTCHA_BORDER_COLOR, "105,179,90"); + // 验证码文本字符颜色 默认为Color.BLACK + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_COLOR, "blue"); + // 验证码图片宽度 默认为200 + properties.setProperty(KAPTCHA_IMAGE_WIDTH, "160"); + // 验证码图片高度 默认为50 + properties.setProperty(KAPTCHA_IMAGE_HEIGHT, "60"); + // 验证码文本字符大小 默认为40 + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_SIZE, "35"); + // KAPTCHA_SESSION_KEY + properties.setProperty(KAPTCHA_SESSION_CONFIG_KEY, "kaptchaCodeMath"); + // 验证码文本生成器 + properties.setProperty(KAPTCHA_TEXTPRODUCER_IMPL, "net.hzvtc.framework.config.KaptchaTextCreator"); + // 验证码文本字符间距 默认为2 + properties.setProperty(KAPTCHA_TEXTPRODUCER_CHAR_SPACE, "3"); + // 验证码文本字符长度 默认为5 + properties.setProperty(KAPTCHA_TEXTPRODUCER_CHAR_LENGTH, "6"); + // 验证码文本字体样式 默认为new Font("Arial", 1, fontSize), new Font("Courier", 1, fontSize) + properties.setProperty(KAPTCHA_TEXTPRODUCER_FONT_NAMES, "Arial,Courier"); + // 验证码噪点颜色 默认为Color.BLACK + properties.setProperty(KAPTCHA_NOISE_COLOR, "white"); + // 干扰实现类 + properties.setProperty(KAPTCHA_NOISE_IMPL, "com.google.code.kaptcha.impl.NoNoise"); + // 图片样式 水纹com.google.code.kaptcha.impl.WaterRipple 鱼眼com.google.code.kaptcha.impl.FishEyeGimpy 阴影com.google.code.kaptcha.impl.ShadowGimpy + properties.setProperty(KAPTCHA_OBSCURIFICATOR_IMPL, "com.google.code.kaptcha.impl.ShadowGimpy"); + Config config = new Config(properties); + defaultKaptcha.setConfig(config); + return defaultKaptcha; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/DruidConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/DruidConfig.java new file mode 100644 index 0000000..f8d0979 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/DruidConfig.java @@ -0,0 +1,127 @@ +package net.hzvtc.framework.config; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.sql.DataSource; + +import net.hzvtc.common.enums.DataSourceType; +import net.hzvtc.common.utils.spring.SpringUtils; +import net.hzvtc.framework.config.properties.DruidProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder; +import com.alibaba.druid.spring.boot.autoconfigure.properties.DruidStatProperties; +import com.alibaba.druid.util.Utils; +import net.hzvtc.framework.datasource.DynamicDataSource; + +/** + * druid 配置多数据源 + * + * @author admin + */ +@Configuration +public class DruidConfig +{ + @Bean + @ConfigurationProperties("spring.datasource.druid.master") + public DataSource masterDataSource(DruidProperties druidProperties) + { + DruidDataSource dataSource = DruidDataSourceBuilder.create().build(); + return druidProperties.dataSource(dataSource); + } + + @Bean + @ConfigurationProperties("spring.datasource.druid.slave") + @ConditionalOnProperty(prefix = "spring.datasource.druid.slave", name = "enabled", havingValue = "true") + public DataSource slaveDataSource(DruidProperties druidProperties) + { + DruidDataSource dataSource = DruidDataSourceBuilder.create().build(); + return druidProperties.dataSource(dataSource); + } + + @Bean(name = "dynamicDataSource") + @Primary + public DynamicDataSource dataSource(DataSource masterDataSource) + { + Map targetDataSources = new HashMap<>(); + targetDataSources.put(DataSourceType.MASTER.name(), masterDataSource); + setDataSource(targetDataSources, DataSourceType.SLAVE.name(), "slaveDataSource"); + return new DynamicDataSource(masterDataSource, targetDataSources); + } + + /** + * 设置数据源 + * + * @param targetDataSources 备选数据源集合 + * @param sourceName 数据源名称 + * @param beanName bean名称 + */ + public void setDataSource(Map targetDataSources, String sourceName, String beanName) + { + try + { + DataSource dataSource = SpringUtils.getBean(beanName); + targetDataSources.put(sourceName, dataSource); + } + catch (Exception e) + { + } + } + + /** + * 去除监控页面底部的广告 + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Bean + @ConditionalOnProperty(name = "spring.datasource.druid.statViewServlet.enabled", havingValue = "true") + public FilterRegistrationBean removeDruidFilterRegistrationBean(DruidStatProperties properties) + { + // 获取web监控页面的参数 + DruidStatProperties.StatViewServlet config = properties.getStatViewServlet(); + // 提取common.js的配置路径 + String pattern = config.getUrlPattern() != null ? config.getUrlPattern() : "/druid/*"; + String commonJsPattern = pattern.replaceAll("\\*", "js/common.js"); + final String filePath = "support/http/resources/js/common.js"; + // 创建filter进行过滤 + Filter filter = new Filter() + { + @Override + public void init(javax.servlet.FilterConfig filterConfig) throws ServletException + { + } + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException + { + chain.doFilter(request, response); + // 重置缓冲区,响应头不会被重置 + response.resetBuffer(); + // 获取common.js + String text = Utils.readFromResource(filePath); + // 正则替换banner, 除去底部的广告信息 + text = text.replaceAll("
", ""); + text = text.replaceAll("powered.*?shrek.wang", ""); + response.getWriter().write(text); + } + @Override + public void destroy() + { + } + }; + FilterRegistrationBean registrationBean = new FilterRegistrationBean(); + registrationBean.setFilter(filter); + registrationBean.addUrlPatterns(commonJsPattern); + return registrationBean; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FastJson2JsonRedisSerializer.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FastJson2JsonRedisSerializer.java new file mode 100644 index 0000000..a91a163 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FastJson2JsonRedisSerializer.java @@ -0,0 +1,62 @@ +package net.hzvtc.framework.config; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.parser.ParserConfig; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.util.Assert; + +import java.nio.charset.Charset; + +/** + * Redis使用FastJson序列化 + * + * @author admin + */ +public class FastJson2JsonRedisSerializer implements RedisSerializer { + public static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8"); + + static { + ParserConfig.getGlobalInstance().setAutoTypeSupport(true); + } + + @SuppressWarnings("unused") + private ObjectMapper objectMapper = new ObjectMapper(); + private Class clazz; + + public FastJson2JsonRedisSerializer(Class clazz) { + super(); + this.clazz = clazz; + } + + @Override + public byte[] serialize(T t) throws SerializationException { + if (t == null) { + return new byte[0]; + } + return JSON.toJSONString(t, SerializerFeature.WriteClassName).getBytes(DEFAULT_CHARSET); + } + + @Override + public T deserialize(byte[] bytes) throws SerializationException { + if (bytes == null || bytes.length <= 0) { + return null; + } + String str = new String(bytes, DEFAULT_CHARSET); + + return JSON.parseObject(str, clazz); + } + + public void setObjectMapper(ObjectMapper objectMapper) { + Assert.notNull(objectMapper, "'objectMapper' must not be null"); + this.objectMapper = objectMapper; + } + + protected JavaType getJavaType(Class clazz) { + return TypeFactory.defaultInstance().constructType(clazz); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FilterConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FilterConfig.java new file mode 100644 index 0000000..79d5100 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/FilterConfig.java @@ -0,0 +1,59 @@ +package net.hzvtc.framework.config; + +import java.util.HashMap; +import java.util.Map; +import javax.servlet.DispatcherType; + +import net.hzvtc.common.filter.RepeatableFilter; +import net.hzvtc.common.filter.XssFilter; +import net.hzvtc.common.utils.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Filter配置 + * + * @author admin + */ +@Configuration +public class FilterConfig +{ + @Value("${xss.excludes}") + private String excludes; + + @Value("${xss.urlPatterns}") + private String urlPatterns; + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Bean + @ConditionalOnProperty(value = "xss.enabled", havingValue = "true") + public FilterRegistrationBean xssFilterRegistration() + { + FilterRegistrationBean registration = new FilterRegistrationBean(); + registration.setDispatcherTypes(DispatcherType.REQUEST); + registration.setFilter(new XssFilter()); + registration.addUrlPatterns(StringUtils.split(urlPatterns, ",")); + registration.setName("xssFilter"); + registration.setOrder(FilterRegistrationBean.HIGHEST_PRECEDENCE); + Map initParameters = new HashMap(); + initParameters.put("excludes", excludes); + registration.setInitParameters(initParameters); + return registration; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Bean + public FilterRegistrationBean someFilterRegistration() + { + FilterRegistrationBean registration = new FilterRegistrationBean(); + registration.setFilter(new RepeatableFilter()); + registration.addUrlPatterns("/*"); + registration.setName("repeatableFilter"); + registration.setOrder(FilterRegistrationBean.LOWEST_PRECEDENCE); + return registration; + } + +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/KaptchaTextCreator.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/KaptchaTextCreator.java new file mode 100644 index 0000000..7a474a9 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/KaptchaTextCreator.java @@ -0,0 +1,75 @@ +package net.hzvtc.framework.config; + +import java.util.Random; +import com.google.code.kaptcha.text.impl.DefaultTextCreator; + +/** + * 验证码文本生成器 + * + * @author admin + */ +public class KaptchaTextCreator extends DefaultTextCreator +{ + private static final String[] CNUMBERS = "0,1,2,3,4,5,6,7,8,9,10".split(","); + + @Override + public String getText() + { + Integer result = 0; + Random random = new Random(); + int x = random.nextInt(10); + int y = random.nextInt(10); + StringBuilder suChinese = new StringBuilder(); + int randomoperands = (int) Math.round(Math.random() * 2); + if (randomoperands == 0) + { + result = x * y; + suChinese.append(CNUMBERS[x]); + suChinese.append("*"); + suChinese.append(CNUMBERS[y]); + } + else if (randomoperands == 1) + { + if (!(x == 0) && y % x == 0) + { + result = y / x; + suChinese.append(CNUMBERS[y]); + suChinese.append("/"); + suChinese.append(CNUMBERS[x]); + } + else + { + result = x + y; + suChinese.append(CNUMBERS[x]); + suChinese.append("+"); + suChinese.append(CNUMBERS[y]); + } + } + else if (randomoperands == 2) + { + if (x >= y) + { + result = x - y; + suChinese.append(CNUMBERS[x]); + suChinese.append("-"); + suChinese.append(CNUMBERS[y]); + } + else + { + result = y - x; + suChinese.append(CNUMBERS[y]); + suChinese.append("-"); + suChinese.append(CNUMBERS[x]); + } + } + else + { + result = x + y; + suChinese.append(CNUMBERS[x]); + suChinese.append("+"); + suChinese.append(CNUMBERS[y]); + } + suChinese.append("=?@" + result); + return suChinese.toString(); + } +} \ No newline at end of file diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/MyBatisConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/MyBatisConfig.java new file mode 100644 index 0000000..23da432 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/MyBatisConfig.java @@ -0,0 +1,133 @@ +package net.hzvtc.framework.config; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import javax.sql.DataSource; + +import net.hzvtc.common.utils.StringUtils; +import org.apache.ibatis.io.VFS; +import org.apache.ibatis.session.SqlSessionFactory; +import org.mybatis.spring.SqlSessionFactoryBean; +import org.mybatis.spring.boot.autoconfigure.SpringBootVFS; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; +import org.springframework.core.type.classreading.CachingMetadataReaderFactory; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.util.ClassUtils; + +/** + * Mybatis支持*匹配扫描包 + * + * @author admin + */ +@Configuration +public class MyBatisConfig +{ + @Autowired + private Environment env; + + static final String DEFAULT_RESOURCE_PATTERN = "**/*.class"; + + public static String setTypeAliasesPackage(String typeAliasesPackage) + { + ResourcePatternResolver resolver = (ResourcePatternResolver) new PathMatchingResourcePatternResolver(); + MetadataReaderFactory metadataReaderFactory = new CachingMetadataReaderFactory(resolver); + List allResult = new ArrayList(); + try + { + for (String aliasesPackage : typeAliasesPackage.split(",")) + { + List result = new ArrayList(); + aliasesPackage = ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX + + ClassUtils.convertClassNameToResourcePath(aliasesPackage.trim()) + "/" + DEFAULT_RESOURCE_PATTERN; + Resource[] resources = resolver.getResources(aliasesPackage); + if (resources != null && resources.length > 0) + { + MetadataReader metadataReader = null; + for (Resource resource : resources) + { + if (resource.isReadable()) + { + metadataReader = metadataReaderFactory.getMetadataReader(resource); + try + { + result.add(Class.forName(metadataReader.getClassMetadata().getClassName()).getPackage().getName()); + } + catch (ClassNotFoundException e) + { + e.printStackTrace(); + } + } + } + } + if (result.size() > 0) + { + HashSet hashResult = new HashSet(result); + allResult.addAll(hashResult); + } + } + if (allResult.size() > 0) + { + typeAliasesPackage = String.join(",", (String[]) allResult.toArray(new String[0])); + } + else + { + throw new RuntimeException("mybatis typeAliasesPackage 路径扫描错误,参数typeAliasesPackage:" + typeAliasesPackage + "未找到任何包"); + } + } + catch (IOException e) + { + e.printStackTrace(); + } + return typeAliasesPackage; + } + + public Resource[] resolveMapperLocations(String[] mapperLocations) + { + ResourcePatternResolver resourceResolver = new PathMatchingResourcePatternResolver(); + List resources = new ArrayList(); + if (mapperLocations != null) + { + for (String mapperLocation : mapperLocations) + { + try + { + Resource[] mappers = resourceResolver.getResources(mapperLocation); + resources.addAll(Arrays.asList(mappers)); + } + catch (IOException e) + { + // ignore + } + } + } + return resources.toArray(new Resource[resources.size()]); + } + + @Bean + public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception + { + String typeAliasesPackage = env.getProperty("mybatis.typeAliasesPackage"); + String mapperLocations = env.getProperty("mybatis.mapperLocations"); + String configLocation = env.getProperty("mybatis.configLocation"); + typeAliasesPackage = setTypeAliasesPackage(typeAliasesPackage); + VFS.addImplClass(SpringBootVFS.class); + + final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); + sessionFactory.setDataSource(dataSource); + sessionFactory.setTypeAliasesPackage(typeAliasesPackage); + sessionFactory.setMapperLocations(resolveMapperLocations(StringUtils.split(mapperLocations, ","))); + sessionFactory.setConfigLocation(new DefaultResourceLoader().getResource(configLocation)); + return sessionFactory.getObject(); + } +} \ No newline at end of file diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/RedisConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/RedisConfig.java new file mode 100644 index 0000000..7d9f8c7 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/RedisConfig.java @@ -0,0 +1,79 @@ +package net.hzvtc.framework.config; + +import org.springframework.cache.annotation.CachingConfigurerSupport; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator; + +/** + * redis配置 + * + * @author admin + */ +@Configuration +@EnableCaching +public class RedisConfig extends CachingConfigurerSupport +{ + @Bean + @SuppressWarnings(value = { "unchecked", "rawtypes" }) + public RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) + { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(connectionFactory); + + FastJson2JsonRedisSerializer serializer = new FastJson2JsonRedisSerializer(Object.class); + + ObjectMapper mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); + mapper.activateDefaultTyping(LaissezFaireSubTypeValidator.instance, ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY); + serializer.setObjectMapper(mapper); + + // 使用StringRedisSerializer来序列化和反序列化redis的key值 + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(serializer); + + // Hash的key也采用StringRedisSerializer的序列化方式 + template.setHashKeySerializer(new StringRedisSerializer()); + template.setHashValueSerializer(serializer); + + template.afterPropertiesSet(); + return template; + } + + @Bean + public DefaultRedisScript limitScript() + { + DefaultRedisScript redisScript = new DefaultRedisScript<>(); + redisScript.setScriptText(limitScriptText()); + redisScript.setResultType(Long.class); + return redisScript; + } + + /** + * 限流脚本 + */ + private String limitScriptText() + { + return "local key = KEYS[1]\n" + + "local count = tonumber(ARGV[1])\n" + + "local time = tonumber(ARGV[2])\n" + + "local current = redis.call('get', key);\n" + + "if current and tonumber(current) > count then\n" + + " return tonumber(current);\n" + + "end\n" + + "current = redis.call('incr', key)\n" + + "if tonumber(current) == 1 then\n" + + " redis.call('expire', key, time)\n" + + "end\n" + + "return tonumber(current);"; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ResourcesConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ResourcesConfig.java new file mode 100644 index 0000000..786d149 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ResourcesConfig.java @@ -0,0 +1,66 @@ +package net.hzvtc.framework.config; + +import net.hzvtc.common.config.AiBirdSysConfig; +import net.hzvtc.common.constant.Constants; +import net.hzvtc.framework.interceptor.RepeatSubmitInterceptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +/** + * 通用配置 + * + * @author admin + */ +@Configuration +public class ResourcesConfig implements WebMvcConfigurer { + @Autowired + private RepeatSubmitInterceptor repeatSubmitInterceptor; + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + /** 本地文件上传路径 */ + registry.addResourceHandler(Constants.RESOURCE_PREFIX + "/**") + .addResourceLocations("file:" + AiBirdSysConfig.getProfile() + "/"); + + /** swagger配置 */ + registry.addResourceHandler("/swagger-ui/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/springfox-swagger-ui/"); + } + + /** + * 自定义拦截规则 + */ + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(repeatSubmitInterceptor).addPathPatterns("/**"); + } + + /** + * 跨域配置 + */ + @Bean + public CorsFilter corsFilter() { + CorsConfiguration config = new CorsConfiguration(); + config.setAllowCredentials(true); + // 设置访问源地址 + config.addAllowedOriginPattern("*"); + // 设置访问源请求头 + config.addAllowedHeader("*"); + // 设置访问源请求方法 + config.addAllowedMethod("*"); + // 有效期 1800秒 + config.setMaxAge(1800L); + // 添加映射路径,拦截一切请求 + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/**", config); + // 返回新的CorsFilter + return new CorsFilter(source); + } +} \ No newline at end of file diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/SecurityConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/SecurityConfig.java new file mode 100644 index 0000000..64bd4c2 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/SecurityConfig.java @@ -0,0 +1,145 @@ +package net.hzvtc.framework.config; + +import net.hzvtc.framework.security.filter.JwtAuthenticationTokenFilter; +import net.hzvtc.framework.security.handle.AuthenticationEntryPointImpl; +import net.hzvtc.framework.security.handle.LogoutSuccessHandlerImpl; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.core.userdetails.UserDetailsService; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; +import org.springframework.security.web.authentication.logout.LogoutFilter; +import org.springframework.web.filter.CorsFilter; + +/** + * spring security配置 + * + * @author admin + */ +@EnableGlobalMethodSecurity(prePostEnabled = true, securedEnabled = true) +public class SecurityConfig extends WebSecurityConfigurerAdapter { + /** + * 自定义用户认证逻辑 + */ + @Autowired + private UserDetailsService userDetailsService; + + /** + * 认证失败处理类 + */ + @Autowired + private AuthenticationEntryPointImpl unauthorizedHandler; + + /** + * 退出处理类 + */ + @Autowired + private LogoutSuccessHandlerImpl logoutSuccessHandler; + + /** + * token认证过滤器 + */ + @Autowired + private JwtAuthenticationTokenFilter authenticationTokenFilter; + + /** + * 跨域过滤器 + */ + @Autowired + private CorsFilter corsFilter; + + /** + * 解决 无法直接注入 AuthenticationManager + * + * @return + * @throws Exception + */ + @Bean + @Override + public AuthenticationManager authenticationManagerBean() throws Exception { + return super.authenticationManagerBean(); + } + + /** + * anyRequest | 匹配所有请求路径 + * access | SpringEl表达式结果为true时可以访问 + * anonymous | 匿名可以访问 + * denyAll | 用户不能访问 + * fullyAuthenticated | 用户完全认证可以访问(非remember-me下自动登录) + * hasAnyAuthority | 如果有参数,参数表示权限,则其中任何一个权限可以访问 + * hasAnyRole | 如果有参数,参数表示角色,则其中任何一个角色可以访问 + * hasAuthority | 如果有参数,参数表示权限,则其权限可以访问 + * hasIpAddress | 如果有参数,参数表示IP地址,如果用户IP和参数匹配,则可以访问 + * hasRole | 如果有参数,参数表示角色,则其角色可以访问 + * permitAll | 用户可以任意访问 + * rememberMe | 允许通过remember-me登录的用户访问 + * authenticated | 用户登录后可访问 + */ + @Override + protected void configure(HttpSecurity httpSecurity) throws Exception { + httpSecurity + // CSRF禁用,因为不使用session + .csrf().disable() + // 认证失败处理类 + .exceptionHandling().authenticationEntryPoint(unauthorizedHandler).and() + // 基于token,所以不需要session + .sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS).and() + // 过滤请求 + .authorizeRequests() + // 对于登录login 注册register 验证码captchaImage 允许匿名访问 + .antMatchers("/login", "/web/api/getToken", + "/appCaptchaImage", + "/captchaImage", + "/appUpdate", + "/sm2login", + "/web/getPublicKey", + "/restPassword").anonymous() + .antMatchers( + HttpMethod.GET, + "/", + "/*.html", + "/**/*.html", + "/**/*.css", + "/**/*.js", + "/profile/**" + ).permitAll() + .antMatchers("/swagger-ui.html").anonymous() + .antMatchers("/swagger-resources/**").anonymous() + .antMatchers("/webjars/**").anonymous() + .antMatchers("/*/api-docs").anonymous() + .antMatchers("/druid/**").anonymous() + // 除上面外的所有请求全部需要鉴权认证 + .anyRequest().authenticated() + .and() + .headers().frameOptions().disable(); + httpSecurity.logout().logoutUrl("/logout").logoutSuccessHandler(logoutSuccessHandler); + // 添加JWT filter + httpSecurity.addFilterBefore(authenticationTokenFilter, UsernamePasswordAuthenticationFilter.class); + // 添加CORS filter + httpSecurity.addFilterBefore(corsFilter, JwtAuthenticationTokenFilter.class); + httpSecurity.addFilterBefore(corsFilter, LogoutFilter.class); + } + + /** + * 强散列哈希加密实现 + */ + @Bean + public BCryptPasswordEncoder bCryptPasswordEncoder() { + return new BCryptPasswordEncoder(); + } + + /** + * 身份认证接口 + */ + @Override + protected void configure(AuthenticationManagerBuilder auth) throws Exception { + auth.userDetailsService(userDetailsService).passwordEncoder(bCryptPasswordEncoder()); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ServerConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ServerConfig.java new file mode 100644 index 0000000..4e111a9 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ServerConfig.java @@ -0,0 +1,33 @@ +package net.hzvtc.framework.config; + +import javax.servlet.http.HttpServletRequest; + +import net.hzvtc.common.utils.ServletUtils; +import org.springframework.stereotype.Component; + +/** + * 服务相关配置 + * + * @author admin + */ +@Component +public class ServerConfig +{ + /** + * 获取完整的请求路径,包括:域名,端口,上下文访问路径 + * + * @return 服务地址 + */ + public String getUrl() + { + HttpServletRequest request = ServletUtils.getRequest(); + return getDomain(request); + } + + public static String getDomain(HttpServletRequest request) + { + StringBuffer url = request.getRequestURL(); + String contextPath = request.getServletContext().getContextPath(); + return url.delete(url.length() - request.getRequestURI().length(), url.length()).append(contextPath).toString(); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ThreadPoolConfig.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ThreadPoolConfig.java new file mode 100644 index 0000000..d95b356 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/ThreadPoolConfig.java @@ -0,0 +1,63 @@ +package net.hzvtc.framework.config; + +import net.hzvtc.common.utils.Threads; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 线程池配置 + * + * @author admin + **/ +@Configuration +public class ThreadPoolConfig +{ + // 核心线程池大小 + private int corePoolSize = 50; + + // 最大可创建的线程数 + private int maxPoolSize = 200; + + // 队列最大长度 + private int queueCapacity = 1000; + + // 线程池维护线程所允许的空闲时间 + private int keepAliveSeconds = 300; + + @Bean(name = "threadPoolTaskExecutor") + public ThreadPoolTaskExecutor threadPoolTaskExecutor() + { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setMaxPoolSize(maxPoolSize); + executor.setCorePoolSize(corePoolSize); + executor.setQueueCapacity(queueCapacity); + executor.setKeepAliveSeconds(keepAliveSeconds); + // 线程池对拒绝任务(无线程可用)的处理策略 + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + return executor; + } + + /** + * 执行周期性或定时任务 + */ + @Bean(name = "scheduledExecutorService") + protected ScheduledExecutorService scheduledExecutorService() + { + return new ScheduledThreadPoolExecutor(corePoolSize, + new BasicThreadFactory.Builder().namingPattern("schedule-pool-%d").daemon(true).build(), + new ThreadPoolExecutor.CallerRunsPolicy()) + { + @Override + protected void afterExecute(Runnable r, Throwable t) + { + super.afterExecute(r, t); + Threads.printException(r, t); + } + }; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/properties/DruidProperties.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/properties/DruidProperties.java new file mode 100644 index 0000000..39a6be2 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/config/properties/DruidProperties.java @@ -0,0 +1,77 @@ +package net.hzvtc.framework.config.properties; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import com.alibaba.druid.pool.DruidDataSource; + +/** + * druid 配置属性 + * + * @author admin + */ +@Configuration +public class DruidProperties +{ + @Value("${spring.datasource.druid.initialSize}") + private int initialSize; + + @Value("${spring.datasource.druid.minIdle}") + private int minIdle; + + @Value("${spring.datasource.druid.maxActive}") + private int maxActive; + + @Value("${spring.datasource.druid.maxWait}") + private int maxWait; + + @Value("${spring.datasource.druid.timeBetweenEvictionRunsMillis}") + private int timeBetweenEvictionRunsMillis; + + @Value("${spring.datasource.druid.minEvictableIdleTimeMillis}") + private int minEvictableIdleTimeMillis; + + @Value("${spring.datasource.druid.maxEvictableIdleTimeMillis}") + private int maxEvictableIdleTimeMillis; + + @Value("${spring.datasource.druid.validationQuery}") + private String validationQuery; + + @Value("${spring.datasource.druid.testWhileIdle}") + private boolean testWhileIdle; + + @Value("${spring.datasource.druid.testOnBorrow}") + private boolean testOnBorrow; + + @Value("${spring.datasource.druid.testOnReturn}") + private boolean testOnReturn; + + public DruidDataSource dataSource(DruidDataSource datasource) + { + /** 配置初始化大小、最小、最大 */ + datasource.setInitialSize(initialSize); + datasource.setMaxActive(maxActive); + datasource.setMinIdle(minIdle); + + /** 配置获取连接等待超时的时间 */ + datasource.setMaxWait(maxWait); + + /** 配置间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 */ + datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + + /** 配置一个连接在池中最小、最大生存的时间,单位是毫秒 */ + datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + datasource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis); + + /** + * 用来检测连接是否有效的sql,要求是一个查询语句,常用select 'x'。如果validationQuery为null,testOnBorrow、testOnReturn、testWhileIdle都不会起作用。 + */ + datasource.setValidationQuery(validationQuery); + /** 建议配置为true,不影响性能,并且保证安全性。申请连接的时候检测,如果空闲时间大于timeBetweenEvictionRunsMillis,执行validationQuery检测连接是否有效。 */ + datasource.setTestWhileIdle(testWhileIdle); + /** 申请连接时执行validationQuery检测连接是否有效,做了这个配置会降低性能。 */ + datasource.setTestOnBorrow(testOnBorrow); + /** 归还连接时执行validationQuery检测连接是否有效,做了这个配置会降低性能。 */ + datasource.setTestOnReturn(testOnReturn); + return datasource; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSource.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSource.java new file mode 100644 index 0000000..0495010 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSource.java @@ -0,0 +1,26 @@ +package net.hzvtc.framework.datasource; + +import java.util.Map; +import javax.sql.DataSource; +import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource; + +/** + * 动态数据源 + * + * @author admin + */ +public class DynamicDataSource extends AbstractRoutingDataSource +{ + public DynamicDataSource(DataSource defaultTargetDataSource, Map targetDataSources) + { + super.setDefaultTargetDataSource(defaultTargetDataSource); + super.setTargetDataSources(targetDataSources); + super.afterPropertiesSet(); + } + + @Override + protected Object determineCurrentLookupKey() + { + return DynamicDataSourceContextHolder.getDataSourceType(); + } +} \ No newline at end of file diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSourceContextHolder.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSourceContextHolder.java new file mode 100644 index 0000000..230c945 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/DynamicDataSourceContextHolder.java @@ -0,0 +1,45 @@ +package net.hzvtc.framework.datasource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 数据源切换处理 + * + * @author admin + */ +public class DynamicDataSourceContextHolder +{ + public static final Logger log = LoggerFactory.getLogger(DynamicDataSourceContextHolder.class); + + /** + * 使用ThreadLocal维护变量,ThreadLocal为每个使用该变量的线程提供独立的变量副本, + * 所以每一个线程都可以独立地改变自己的副本,而不会影响其它线程所对应的副本。 + */ + private static final ThreadLocal CONTEXT_HOLDER = new ThreadLocal<>(); + + /** + * 设置数据源的变量 + */ + public static void setDataSourceType(String dsType) + { + log.info("切换到{}数据源", dsType); + CONTEXT_HOLDER.set(dsType); + } + + /** + * 获得数据源的变量 + */ + public static String getDataSourceType() + { + return CONTEXT_HOLDER.get(); + } + + /** + * 清空数据源变量 + */ + public static void clearDataSourceType() + { + CONTEXT_HOLDER.remove(); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/MySqlDialectCustom.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/MySqlDialectCustom.java new file mode 100644 index 0000000..3501f34 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/datasource/MySqlDialectCustom.java @@ -0,0 +1,24 @@ +package net.hzvtc.framework.datasource; + +import com.github.pagehelper.Page; +import com.github.pagehelper.dialect.helper.MySqlDialect; +import org.apache.ibatis.cache.CacheKey; + +import java.util.regex.Pattern; + +public class MySqlDialectCustom extends MySqlDialect { + @Override + public String getPageSql(String sql, Page page, CacheKey pageKey) { + Pattern compileFixed = Pattern.compile("/\\*fixed\\*/"); + String compileLimit = "/\\*limit\\*/"; + if (compileFixed.matcher(sql).find()) { + if (page.getStartRow() == 0) { + sql = sql.replaceFirst(compileLimit, " LIMIT ? "); + } else { + sql = sql.replaceFirst(compileLimit, " LIMIT ?, ? "); + } + return sql; + } + return super.getPageSql(sql, page, pageKey); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/RepeatSubmitInterceptor.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/RepeatSubmitInterceptor.java new file mode 100644 index 0000000..884a2b8 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/RepeatSubmitInterceptor.java @@ -0,0 +1,56 @@ +package net.hzvtc.framework.interceptor; + +import java.lang.reflect.Method; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.utils.ServletUtils; +import org.springframework.stereotype.Component; +import org.springframework.web.method.HandlerMethod; +import org.springframework.web.servlet.HandlerInterceptor; +import com.alibaba.fastjson.JSONObject; +import net.hzvtc.common.annotation.RepeatSubmit; + +/** + * 防止重复提交拦截器 + * + * @author admin + */ +@Component +public abstract class RepeatSubmitInterceptor implements HandlerInterceptor +{ + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception + { + if (handler instanceof HandlerMethod) + { + HandlerMethod handlerMethod = (HandlerMethod) handler; + Method method = handlerMethod.getMethod(); + RepeatSubmit annotation = method.getAnnotation(RepeatSubmit.class); + if (annotation != null) + { + if (this.isRepeatSubmit(request, annotation)) + { + AjaxResult ajaxResult = AjaxResult.error(annotation.message()); + ServletUtils.renderString(response, JSONObject.toJSONString(ajaxResult)); + return false; + } + } + return true; + } + else + { + return true; + } + } + + /** + * 验证是否重复提交由子类实现具体的防重复提交的规则 + * + * @param request + * @return + * @throws Exception + */ + public abstract boolean isRepeatSubmit(HttpServletRequest request, RepeatSubmit annotation); +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/impl/SameUrlDataInterceptor.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/impl/SameUrlDataInterceptor.java new file mode 100644 index 0000000..91c95c1 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/interceptor/impl/SameUrlDataInterceptor.java @@ -0,0 +1,111 @@ +package net.hzvtc.framework.interceptor.impl; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.servlet.http.HttpServletRequest; + +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.core.redis.RedisCache; +import net.hzvtc.common.filter.RepeatedlyRequestWrapper; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.common.utils.http.HttpHelper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import com.alibaba.fastjson.JSONObject; +import net.hzvtc.common.annotation.RepeatSubmit; +import net.hzvtc.framework.interceptor.RepeatSubmitInterceptor; + +/** + * 判断请求url和数据是否和上一次相同, + * 如果和上次相同,则是重复提交表单。 有效时间为10秒内。 + * + * @author admin + */ +@Component +public class SameUrlDataInterceptor extends RepeatSubmitInterceptor +{ + public final String REPEAT_PARAMS = "repeatParams"; + + public final String REPEAT_TIME = "repeatTime"; + + // 令牌自定义标识 + @Value("${token.header}") + private String header; + + @Autowired + private RedisCache redisCache; + + @SuppressWarnings("unchecked") + @Override + public boolean isRepeatSubmit(HttpServletRequest request, RepeatSubmit annotation) + { + String nowParams = ""; + if (request instanceof RepeatedlyRequestWrapper) + { + RepeatedlyRequestWrapper repeatedlyRequest = (RepeatedlyRequestWrapper) request; + nowParams = HttpHelper.getBodyString(repeatedlyRequest); + } + + // body参数为空,获取Parameter的数据 + if (StringUtils.isEmpty(nowParams)) + { + nowParams = JSONObject.toJSONString(request.getParameterMap()); + } + Map nowDataMap = new HashMap(); + nowDataMap.put(REPEAT_PARAMS, nowParams); + nowDataMap.put(REPEAT_TIME, System.currentTimeMillis()); + + // 请求地址(作为存放cache的key值) + String url = request.getRequestURI(); + + // 唯一值(没有消息头则使用请求地址) + String submitKey = StringUtils.trimToEmpty(request.getHeader(header)); + + // 唯一标识(指定key + url + 消息头) + String cacheRepeatKey = Constants.REPEAT_SUBMIT_KEY + url + submitKey; + + Object sessionObj = redisCache.getCacheObject(cacheRepeatKey); + if (sessionObj != null) + { + Map sessionMap = (Map) sessionObj; + if (sessionMap.containsKey(url)) + { + Map preDataMap = (Map) sessionMap.get(url); + if (compareParams(nowDataMap, preDataMap) && compareTime(nowDataMap, preDataMap, annotation.interval())) + { + return true; + } + } + } + Map cacheMap = new HashMap(); + cacheMap.put(url, nowDataMap); + redisCache.setCacheObject(cacheRepeatKey, cacheMap, annotation.interval(), TimeUnit.MILLISECONDS); + return false; + } + + /** + * 判断参数是否相同 + */ + private boolean compareParams(Map nowMap, Map preMap) + { + String nowParams = (String) nowMap.get(REPEAT_PARAMS); + String preParams = (String) preMap.get(REPEAT_PARAMS); + return nowParams.equals(preParams); + } + + /** + * 判断两次间隔时间 + */ + private boolean compareTime(Map nowMap, Map preMap, int interval) + { + long time1 = (Long) nowMap.get(REPEAT_TIME); + long time2 = (Long) preMap.get(REPEAT_TIME); + if ((time1 - time2) < interval) + { + return true; + } + return false; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/AsyncManager.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/AsyncManager.java new file mode 100644 index 0000000..de071ce --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/AsyncManager.java @@ -0,0 +1,55 @@ +package net.hzvtc.framework.manager; + +import java.util.TimerTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import net.hzvtc.common.utils.Threads; +import net.hzvtc.common.utils.spring.SpringUtils; + +/** + * 异步任务管理器 + * + * @author admin + */ +public class AsyncManager +{ + /** + * 操作延迟10毫秒 + */ + private final int OPERATE_DELAY_TIME = 10; + + /** + * 异步操作任务调度线程池 + */ + private ScheduledExecutorService executor = SpringUtils.getBean("scheduledExecutorService"); + + /** + * 单例模式 + */ + private AsyncManager(){} + + private static AsyncManager me = new AsyncManager(); + + public static AsyncManager me() + { + return me; + } + + /** + * 执行任务 + * + * @param task 任务 + */ + public void execute(TimerTask task) + { + executor.schedule(task, OPERATE_DELAY_TIME, TimeUnit.MILLISECONDS); + } + + /** + * 停止任务线程池 + */ + public void shutdown() + { + Threads.shutdownAndAwaitTermination(executor); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/ShutdownManager.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/ShutdownManager.java new file mode 100644 index 0000000..d1c373a --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/ShutdownManager.java @@ -0,0 +1,39 @@ +package net.hzvtc.framework.manager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import javax.annotation.PreDestroy; + +/** + * 确保应用退出时能关闭后台线程 + * + * @author admin + */ +@Component +public class ShutdownManager +{ + private static final Logger logger = LoggerFactory.getLogger("sys-user"); + + @PreDestroy + public void destroy() + { + shutdownAsyncManager(); + } + + /** + * 停止异步执行任务 + */ + private void shutdownAsyncManager() + { + try + { + logger.info("====关闭后台任务任务线程池===="); + AsyncManager.me().shutdown(); + } + catch (Exception e) + { + logger.error(e.getMessage(), e); + } + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/factory/AsyncFactory.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/factory/AsyncFactory.java new file mode 100644 index 0000000..ba9af82 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/manager/factory/AsyncFactory.java @@ -0,0 +1,103 @@ +package net.hzvtc.framework.manager.factory; + +import java.util.TimerTask; + +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.utils.LogUtils; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.common.utils.ip.AddressUtils; +import net.hzvtc.common.utils.ip.IpUtils; +import net.hzvtc.common.utils.spring.SpringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import net.hzvtc.system.domain.SysLogininfor; +import net.hzvtc.system.domain.SysOperLog; +import net.hzvtc.system.service.SysLogininforService; +import net.hzvtc.system.service.SysOperLogService; +import eu.bitwalker.useragentutils.UserAgent; + +/** + * 异步工厂(产生任务用) + * + * @author admin + */ +public class AsyncFactory +{ + private static final Logger sys_user_logger = LoggerFactory.getLogger("sys-user"); + + /** + * 记录登录信息 + * + * @param username 用户名 + * @param status 状态 + * @param message 消息 + * @param args 列表 + * @return 任务task + */ + public static TimerTask recordLogininfor(final String username, final String status, final String message, + final Object... args) + { + final UserAgent userAgent = UserAgent.parseUserAgentString(ServletUtils.getRequest().getHeader("User-Agent")); + final String ip = IpUtils.getIpAddr(ServletUtils.getRequest()); + return new TimerTask() + { + @Override + public void run() + { + String address = AddressUtils.getRealAddressByIP(ip); + StringBuilder s = new StringBuilder(); + s.append(LogUtils.getBlock(ip)); + s.append(address); + s.append(LogUtils.getBlock(username)); + s.append(LogUtils.getBlock(status)); + s.append(LogUtils.getBlock(message)); + // 打印信息到日志 + sys_user_logger.info(s.toString(), args); + // 获取客户端操作系统 + String os = userAgent.getOperatingSystem().getName(); + // 获取客户端浏览器 + String browser = userAgent.getBrowser().getName(); + // 封装对象 + SysLogininfor logininfor = new SysLogininfor(); + logininfor.setUserName(username); + logininfor.setIpaddr(ip); + logininfor.setLoginLocation(address); + logininfor.setBrowser(browser); + logininfor.setOs(os); + logininfor.setMsg(message); + // 日志状态 + if (StringUtils.equalsAny(status, Constants.LOGIN_SUCCESS, Constants.LOGOUT, Constants.REGISTER)) + { + logininfor.setStatus(Constants.SUCCESS); + } + else if (Constants.LOGIN_FAIL.equals(status)) + { + logininfor.setStatus(Constants.FAIL); + } + // 插入数据 + SpringUtils.getBean(SysLogininforService.class).insertLogininfor(logininfor); + } + }; + } + + /** + * 操作日志记录 + * + * @param operLog 操作日志信息 + * @return 任务task + */ + public static TimerTask recordOper(final SysOperLog operLog) + { + return new TimerTask() + { + @Override + public void run() + { + // 远程查询操作地点 + operLog.setOperLocation(AddressUtils.getRealAddressByIP(operLog.getOperIp())); + SpringUtils.getBean(SysOperLogService.class).insertOperlog(operLog); + } + }; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/filter/JwtAuthenticationTokenFilter.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/filter/JwtAuthenticationTokenFilter.java new file mode 100644 index 0000000..37dbe19 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/filter/JwtAuthenticationTokenFilter.java @@ -0,0 +1,45 @@ +package net.hzvtc.framework.security.filter; + +import java.io.IOException; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.web.authentication.WebAuthenticationDetailsSource; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; +import net.hzvtc.framework.web.service.TokenService; + +/** + * token过滤器 验证token有效性 + * + * @author admin + */ +@Component +public class JwtAuthenticationTokenFilter extends OncePerRequestFilter +{ + @Autowired + private TokenService tokenService; + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException + { + LoginUser loginUser = tokenService.getLoginUser(request); + if (StringUtils.isNotNull(loginUser) && StringUtils.isNull(SecurityUtils.getAuthentication())) + { + tokenService.verifyToken(loginUser); + UsernamePasswordAuthenticationToken authenticationToken = new UsernamePasswordAuthenticationToken(loginUser, null, loginUser.getAuthorities()); + authenticationToken.setDetails(new WebAuthenticationDetailsSource().buildDetails(request)); + SecurityContextHolder.getContext().setAuthentication(authenticationToken); + } + chain.doFilter(request, response); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/AuthenticationEntryPointImpl.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/AuthenticationEntryPointImpl.java new file mode 100644 index 0000000..4f635af --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/AuthenticationEntryPointImpl.java @@ -0,0 +1,35 @@ +package net.hzvtc.framework.security.handle; + +import java.io.IOException; +import java.io.Serializable; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.hzvtc.common.constant.HttpStatus; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.web.AuthenticationEntryPoint; +import org.springframework.stereotype.Component; +import com.alibaba.fastjson.JSON; + +/** + * 认证失败处理类 返回未授权 + * + * @author admin + */ +@Component +public class AuthenticationEntryPointImpl implements AuthenticationEntryPoint, Serializable +{ + private static final long serialVersionUID = -8970718410437077606L; + + @Override + public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException e) + throws IOException + { + int code = HttpStatus.UNAUTHORIZED; + String msg = StringUtils.format("请求访问:{},认证失败,无法访问系统资源", request.getRequestURI()); + ServletUtils.renderString(response, JSON.toJSONString(AjaxResult.error(code, msg))); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/LogoutSuccessHandlerImpl.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/LogoutSuccessHandlerImpl.java new file mode 100644 index 0000000..f8b0581 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/security/handle/LogoutSuccessHandlerImpl.java @@ -0,0 +1,54 @@ +package net.hzvtc.framework.security.handle; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.constant.HttpStatus; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.core.Authentication; +import org.springframework.security.web.authentication.logout.LogoutSuccessHandler; +import com.alibaba.fastjson.JSON; +import net.hzvtc.framework.manager.AsyncManager; +import net.hzvtc.framework.manager.factory.AsyncFactory; +import net.hzvtc.framework.web.service.TokenService; + +/** + * 自定义退出处理类 返回成功 + * + * @author admin + */ +@Configuration +public class LogoutSuccessHandlerImpl implements LogoutSuccessHandler +{ + @Autowired + private TokenService tokenService; + + /** + * 退出处理 + * + * @return + */ + @Override + public void onLogoutSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) + throws IOException, ServletException + { + LoginUser loginUser = tokenService.getLoginUser(request); + if (StringUtils.isNotNull(loginUser)) + { + String userName = loginUser.getUsername(); + // 删除用户缓存记录 + tokenService.delLoginUser(loginUser.getToken()); + // 记录用户退出日志 + AsyncManager.me().execute(AsyncFactory.recordLogininfor(userName, Constants.LOGOUT, "退出成功")); + } + ServletUtils.renderString(response, JSON.toJSONString(AjaxResult.error(HttpStatus.SUCCESS, "退出成功"))); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/Server.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/Server.java new file mode 100644 index 0000000..a08a779 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/Server.java @@ -0,0 +1,240 @@ +package net.hzvtc.framework.web.domain; + +import java.net.UnknownHostException; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import net.hzvtc.common.utils.Arith; +import net.hzvtc.common.utils.ip.IpUtils; +import net.hzvtc.framework.web.domain.server.Cpu; +import net.hzvtc.framework.web.domain.server.Jvm; +import net.hzvtc.framework.web.domain.server.Mem; +import net.hzvtc.framework.web.domain.server.Sys; +import net.hzvtc.framework.web.domain.server.SysFile; +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.hardware.CentralProcessor.TickType; +import oshi.hardware.GlobalMemory; +import oshi.hardware.HardwareAbstractionLayer; +import oshi.software.os.FileSystem; +import oshi.software.os.OSFileStore; +import oshi.software.os.OperatingSystem; +import oshi.util.Util; + +/** + * 服务器相关信息 + * + * @author admin + */ +public class Server +{ + private static final int OSHI_WAIT_SECOND = 1000; + + /** + * CPU相关信息 + */ + private Cpu cpu = new Cpu(); + + /** + * 內存相关信息 + */ + private Mem mem = new Mem(); + + /** + * JVM相关信息 + */ + private Jvm jvm = new Jvm(); + + /** + * 服务器相关信息 + */ + private Sys sys = new Sys(); + + /** + * 磁盘相关信息 + */ + private List sysFiles = new LinkedList(); + + public Cpu getCpu() + { + return cpu; + } + + public void setCpu(Cpu cpu) + { + this.cpu = cpu; + } + + public Mem getMem() + { + return mem; + } + + public void setMem(Mem mem) + { + this.mem = mem; + } + + public Jvm getJvm() + { + return jvm; + } + + public void setJvm(Jvm jvm) + { + this.jvm = jvm; + } + + public Sys getSys() + { + return sys; + } + + public void setSys(Sys sys) + { + this.sys = sys; + } + + public List getSysFiles() + { + return sysFiles; + } + + public void setSysFiles(List sysFiles) + { + this.sysFiles = sysFiles; + } + + public void copyTo() throws Exception + { + SystemInfo si = new SystemInfo(); + HardwareAbstractionLayer hal = si.getHardware(); + + setCpuInfo(hal.getProcessor()); + + setMemInfo(hal.getMemory()); + + setSysInfo(); + + setJvmInfo(); + + setSysFiles(si.getOperatingSystem()); + } + + /** + * 设置CPU信息 + */ + private void setCpuInfo(CentralProcessor processor) + { + // CPU信息 + long[] prevTicks = processor.getSystemCpuLoadTicks(); + Util.sleep(OSHI_WAIT_SECOND); + long[] ticks = processor.getSystemCpuLoadTicks(); + long nice = ticks[TickType.NICE.getIndex()] - prevTicks[TickType.NICE.getIndex()]; + long irq = ticks[TickType.IRQ.getIndex()] - prevTicks[TickType.IRQ.getIndex()]; + long softirq = ticks[TickType.SOFTIRQ.getIndex()] - prevTicks[TickType.SOFTIRQ.getIndex()]; + long steal = ticks[TickType.STEAL.getIndex()] - prevTicks[TickType.STEAL.getIndex()]; + long cSys = ticks[TickType.SYSTEM.getIndex()] - prevTicks[TickType.SYSTEM.getIndex()]; + long user = ticks[TickType.USER.getIndex()] - prevTicks[TickType.USER.getIndex()]; + long iowait = ticks[TickType.IOWAIT.getIndex()] - prevTicks[TickType.IOWAIT.getIndex()]; + long idle = ticks[TickType.IDLE.getIndex()] - prevTicks[TickType.IDLE.getIndex()]; + long totalCpu = user + nice + cSys + idle + iowait + irq + softirq + steal; + cpu.setCpuNum(processor.getLogicalProcessorCount()); + cpu.setTotal(totalCpu); + cpu.setSys(cSys); + cpu.setUsed(user); + cpu.setWait(iowait); + cpu.setFree(idle); + } + + /** + * 设置内存信息 + */ + private void setMemInfo(GlobalMemory memory) + { + mem.setTotal(memory.getTotal()); + mem.setUsed(memory.getTotal() - memory.getAvailable()); + mem.setFree(memory.getAvailable()); + } + + /** + * 设置服务器信息 + */ + private void setSysInfo() + { + Properties props = System.getProperties(); + sys.setComputerName(IpUtils.getHostName()); + sys.setComputerIp(IpUtils.getHostIp()); + sys.setOsName(props.getProperty("os.name")); + sys.setOsArch(props.getProperty("os.arch")); + sys.setUserDir(props.getProperty("user.dir")); + } + + /** + * 设置Java虚拟机 + */ + private void setJvmInfo() throws UnknownHostException + { + Properties props = System.getProperties(); + jvm.setTotal(Runtime.getRuntime().totalMemory()); + jvm.setMax(Runtime.getRuntime().maxMemory()); + jvm.setFree(Runtime.getRuntime().freeMemory()); + jvm.setVersion(props.getProperty("java.version")); + jvm.setHome(props.getProperty("java.home")); + } + + /** + * 设置磁盘信息 + */ + private void setSysFiles(OperatingSystem os) + { + FileSystem fileSystem = os.getFileSystem(); + List fsArray = fileSystem.getFileStores(); + for (OSFileStore fs : fsArray) + { + long free = fs.getUsableSpace(); + long total = fs.getTotalSpace(); + long used = total - free; + SysFile sysFile = new SysFile(); + sysFile.setDirName(fs.getMount()); + sysFile.setSysTypeName(fs.getType()); + sysFile.setTypeName(fs.getName()); + sysFile.setTotal(convertFileSize(total)); + sysFile.setFree(convertFileSize(free)); + sysFile.setUsed(convertFileSize(used)); + sysFile.setUsage(Arith.mul(Arith.div(used, total, 4), 100)); + sysFiles.add(sysFile); + } + } + + /** + * 字节转换 + * + * @param size 字节大小 + * @return 转换后值 + */ + public String convertFileSize(long size) + { + long kb = 1024; + long mb = kb * 1024; + long gb = mb * 1024; + if (size >= gb) + { + return String.format("%.1f GB", (float) size / gb); + } + else if (size >= mb) + { + float f = (float) size / mb; + return String.format(f > 100 ? "%.0f MB" : "%.1f MB", f); + } + else if (size >= kb) + { + float f = (float) size / kb; + return String.format(f > 100 ? "%.0f KB" : "%.1f KB", f); + } + else + { + return String.format("%d B", size); + } + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Cpu.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Cpu.java new file mode 100644 index 0000000..0f3e4fd --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Cpu.java @@ -0,0 +1,101 @@ +package net.hzvtc.framework.web.domain.server; + +import net.hzvtc.common.utils.Arith; + +/** + * CPU相关信息 + * + * @author admin + */ +public class Cpu +{ + /** + * 核心数 + */ + private int cpuNum; + + /** + * CPU总的使用率 + */ + private double total; + + /** + * CPU系统使用率 + */ + private double sys; + + /** + * CPU用户使用率 + */ + private double used; + + /** + * CPU当前等待率 + */ + private double wait; + + /** + * CPU当前空闲率 + */ + private double free; + + public int getCpuNum() + { + return cpuNum; + } + + public void setCpuNum(int cpuNum) + { + this.cpuNum = cpuNum; + } + + public double getTotal() + { + return Arith.round(Arith.mul(total, 100), 2); + } + + public void setTotal(double total) + { + this.total = total; + } + + public double getSys() + { + return Arith.round(Arith.mul(sys / total, 100), 2); + } + + public void setSys(double sys) + { + this.sys = sys; + } + + public double getUsed() + { + return Arith.round(Arith.mul(used / total, 100), 2); + } + + public void setUsed(double used) + { + this.used = used; + } + + public double getWait() + { + return Arith.round(Arith.mul(wait / total, 100), 2); + } + + public void setWait(double wait) + { + this.wait = wait; + } + + public double getFree() + { + return Arith.round(Arith.mul(free / total, 100), 2); + } + + public void setFree(double free) + { + this.free = free; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Jvm.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Jvm.java new file mode 100644 index 0000000..73d4ede --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Jvm.java @@ -0,0 +1,122 @@ +package net.hzvtc.framework.web.domain.server; + +import java.lang.management.ManagementFactory; +import net.hzvtc.common.utils.Arith; +import net.hzvtc.common.utils.DateUtils; + +/** + * JVM相关信息 + * + * @author admin + */ +public class Jvm +{ + /** + * 当前JVM占用的内存总数(M) + */ + private double total; + + /** + * JVM最大可用内存总数(M) + */ + private double max; + + /** + * JVM空闲内存(M) + */ + private double free; + + /** + * JDK版本 + */ + private String version; + + /** + * JDK路径 + */ + private String home; + + public double getTotal() + { + return Arith.div(total, (1024 * 1024), 2); + } + + public void setTotal(double total) + { + this.total = total; + } + + public double getMax() + { + return Arith.div(max, (1024 * 1024), 2); + } + + public void setMax(double max) + { + this.max = max; + } + + public double getFree() + { + return Arith.div(free, (1024 * 1024), 2); + } + + public void setFree(double free) + { + this.free = free; + } + + public double getUsed() + { + return Arith.div(total - free, (1024 * 1024), 2); + } + + public double getUsage() + { + return Arith.mul(Arith.div(total - free, total, 4), 100); + } + + /** + * 获取JDK名称 + */ + public String getName() + { + return ManagementFactory.getRuntimeMXBean().getVmName(); + } + + public String getVersion() + { + return version; + } + + public void setVersion(String version) + { + this.version = version; + } + + public String getHome() + { + return home; + } + + public void setHome(String home) + { + this.home = home; + } + + /** + * JDK启动时间 + */ + public String getStartTime() + { + return DateUtils.parseDateToStr(DateUtils.YYYY_MM_DD_HH_MM_SS, DateUtils.getServerStartDate()); + } + + /** + * JDK运行时间 + */ + public String getRunTime() + { + return DateUtils.getDatePoor(DateUtils.getNowDate(), DateUtils.getServerStartDate()); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Mem.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Mem.java new file mode 100644 index 0000000..01e7665 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Mem.java @@ -0,0 +1,61 @@ +package net.hzvtc.framework.web.domain.server; + +import net.hzvtc.common.utils.Arith; + +/** + * 內存相关信息 + * + * @author admin + */ +public class Mem +{ + /** + * 内存总量 + */ + private double total; + + /** + * 已用内存 + */ + private double used; + + /** + * 剩余内存 + */ + private double free; + + public double getTotal() + { + return Arith.div(total, (1024 * 1024 * 1024), 2); + } + + public void setTotal(long total) + { + this.total = total; + } + + public double getUsed() + { + return Arith.div(used, (1024 * 1024 * 1024), 2); + } + + public void setUsed(long used) + { + this.used = used; + } + + public double getFree() + { + return Arith.div(free, (1024 * 1024 * 1024), 2); + } + + public void setFree(long free) + { + this.free = free; + } + + public double getUsage() + { + return Arith.mul(Arith.div(used, total, 4), 100); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Sys.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Sys.java new file mode 100644 index 0000000..88fc983 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/Sys.java @@ -0,0 +1,84 @@ +package net.hzvtc.framework.web.domain.server; + +/** + * 系统相关信息 + * + * @author admin + */ +public class Sys +{ + /** + * 服务器名称 + */ + private String computerName; + + /** + * 服务器Ip + */ + private String computerIp; + + /** + * 项目路径 + */ + private String userDir; + + /** + * 操作系统 + */ + private String osName; + + /** + * 系统架构 + */ + private String osArch; + + public String getComputerName() + { + return computerName; + } + + public void setComputerName(String computerName) + { + this.computerName = computerName; + } + + public String getComputerIp() + { + return computerIp; + } + + public void setComputerIp(String computerIp) + { + this.computerIp = computerIp; + } + + public String getUserDir() + { + return userDir; + } + + public void setUserDir(String userDir) + { + this.userDir = userDir; + } + + public String getOsName() + { + return osName; + } + + public void setOsName(String osName) + { + this.osName = osName; + } + + public String getOsArch() + { + return osArch; + } + + public void setOsArch(String osArch) + { + this.osArch = osArch; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/SysFile.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/SysFile.java new file mode 100644 index 0000000..eaafd00 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/domain/server/SysFile.java @@ -0,0 +1,114 @@ +package net.hzvtc.framework.web.domain.server; + +/** + * 系统文件相关信息 + * + * @author admin + */ +public class SysFile +{ + /** + * 盘符路径 + */ + private String dirName; + + /** + * 盘符类型 + */ + private String sysTypeName; + + /** + * 文件类型 + */ + private String typeName; + + /** + * 总大小 + */ + private String total; + + /** + * 剩余大小 + */ + private String free; + + /** + * 已经使用量 + */ + private String used; + + /** + * 资源的使用率 + */ + private double usage; + + public String getDirName() + { + return dirName; + } + + public void setDirName(String dirName) + { + this.dirName = dirName; + } + + public String getSysTypeName() + { + return sysTypeName; + } + + public void setSysTypeName(String sysTypeName) + { + this.sysTypeName = sysTypeName; + } + + public String getTypeName() + { + return typeName; + } + + public void setTypeName(String typeName) + { + this.typeName = typeName; + } + + public String getTotal() + { + return total; + } + + public void setTotal(String total) + { + this.total = total; + } + + public String getFree() + { + return free; + } + + public void setFree(String free) + { + this.free = free; + } + + public String getUsed() + { + return used; + } + + public void setUsed(String used) + { + this.used = used; + } + + public double getUsage() + { + return usage; + } + + public void setUsage(double usage) + { + this.usage = usage; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/exception/GlobalExceptionHandler.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/exception/GlobalExceptionHandler.java new file mode 100644 index 0000000..5fd6847 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/exception/GlobalExceptionHandler.java @@ -0,0 +1,106 @@ +package net.hzvtc.framework.web.exception; + +import net.hzvtc.common.constant.HttpStatus; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.exception.DemoModeException; +import net.hzvtc.common.exception.ServiceException; +import net.hzvtc.common.utils.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.validation.BindException; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import javax.servlet.http.HttpServletRequest; + +/** + * 全局异常处理器 + * + * @author admin + */ +@RestControllerAdvice +public class GlobalExceptionHandler { + private static final Logger log = LoggerFactory.getLogger(GlobalExceptionHandler.class); + + /** + * 权限校验异常 + */ + @ExceptionHandler(AccessDeniedException.class) + public AjaxResult handleAccessDeniedException(AccessDeniedException e, HttpServletRequest request) { + String requestURI = request.getRequestURI(); + log.error("请求地址'{}',权限校验失败'{}'", requestURI, e.getMessage()); + return AjaxResult.error(HttpStatus.FORBIDDEN, "没有权限,请联系管理员授权"); + } + + /** + * 请求方式不支持 + */ + @ExceptionHandler(HttpRequestMethodNotSupportedException.class) + public AjaxResult handleHttpRequestMethodNotSupported(HttpRequestMethodNotSupportedException e, + HttpServletRequest request) { + String requestURI = request.getRequestURI(); + log.error("请求地址'{}',不支持'{}'请求", requestURI, e.getMethod()); + return AjaxResult.error(e.getMessage()); + } + + /** + * 业务异常 + */ + @ExceptionHandler(ServiceException.class) + public AjaxResult handleServiceException(ServiceException e, HttpServletRequest request) { + log.error(e.getMessage(), e); + Integer code = e.getCode(); + return StringUtils.isNotNull(code) ? AjaxResult.error(code, e.getMessage()) : AjaxResult.error(e.getMessage()); + } + + /** + * 拦截未知的运行时异常 + */ + @ExceptionHandler(RuntimeException.class) + public AjaxResult handleRuntimeException(RuntimeException e, HttpServletRequest request) { + String requestURI = request.getRequestURI(); + log.error("请求地址'{}',发生未知异常.", requestURI, e); + return AjaxResult.error(e.getMessage()); + } + + /** + * 系统异常 + */ + @ExceptionHandler(Exception.class) + public AjaxResult handleException(Exception e, HttpServletRequest request) { + String requestURI = request.getRequestURI(); + log.error("请求地址'{}',发生系统异常.", requestURI, e); + return AjaxResult.error(e.getMessage()); + } + + /** + * 自定义验证异常 + */ + @ExceptionHandler(BindException.class) + public AjaxResult handleBindException(BindException e) { + log.error(e.getMessage(), e); + String message = e.getAllErrors().get(0).getDefaultMessage(); + return AjaxResult.error(message); + } + + /** + * 自定义验证异常 + */ + @ExceptionHandler(MethodArgumentNotValidException.class) + public Object handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { + log.error(e.getMessage(), e); + String message = e.getBindingResult().getFieldError().getDefaultMessage(); + return AjaxResult.error(message); + } + + /** + * 演示模式异常 + */ + @ExceptionHandler(DemoModeException.class) + public AjaxResult handleDemoModeException(DemoModeException e) { + return AjaxResult.error("演示模式,不允许操作"); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/PermissionService.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/PermissionService.java new file mode 100644 index 0000000..8177978 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/PermissionService.java @@ -0,0 +1,166 @@ +package net.hzvtc.framework.web.service; + +import java.util.Set; + +import net.hzvtc.common.core.domain.entity.SysRole; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +/** + * 首创 自定义权限实现,ss取自SpringSecurity首字母 + * + * @author admin + */ +@Service("ss") +public class PermissionService +{ + /** 所有权限标识 */ + private static final String ALL_PERMISSION = "*:*:*"; + + /** 管理员角色权限标识 */ + private static final String SUPER_ADMIN = "admin"; + + private static final String ROLE_DELIMETER = ","; + + private static final String PERMISSION_DELIMETER = ","; + + /** + * 验证用户是否具备某权限 + * + * @param permission 权限字符串 + * @return 用户是否具备某权限 + */ + public boolean hasPermi(String permission) + { + if (StringUtils.isEmpty(permission)) + { + return false; + } + LoginUser loginUser = SecurityUtils.getLoginUser(); + if (StringUtils.isNull(loginUser) || CollectionUtils.isEmpty(loginUser.getPermissions())) + { + return false; + } + return hasPermissions(loginUser.getPermissions(), permission); + } + + /** + * 验证用户是否不具备某权限,与 hasPermi逻辑相反 + * + * @param permission 权限字符串 + * @return 用户是否不具备某权限 + */ + public boolean lacksPermi(String permission) + { + return hasPermi(permission) != true; + } + + /** + * 验证用户是否具有以下任意一个权限 + * + * @param permissions 以 PERMISSION_NAMES_DELIMETER 为分隔符的权限列表 + * @return 用户是否具有以下任意一个权限 + */ + public boolean hasAnyPermi(String permissions) + { + if (StringUtils.isEmpty(permissions)) + { + return false; + } + LoginUser loginUser = SecurityUtils.getLoginUser(); + if (StringUtils.isNull(loginUser) || CollectionUtils.isEmpty(loginUser.getPermissions())) + { + return false; + } + Set authorities = loginUser.getPermissions(); + for (String permission : permissions.split(PERMISSION_DELIMETER)) + { + if (permission != null && hasPermissions(authorities, permission)) + { + return true; + } + } + return false; + } + + /** + * 判断用户是否拥有某个角色 + * + * @param role 角色字符串 + * @return 用户是否具备某角色 + */ + public boolean hasRole(String role) + { + if (StringUtils.isEmpty(role)) + { + return false; + } + LoginUser loginUser = SecurityUtils.getLoginUser(); + if (StringUtils.isNull(loginUser) || CollectionUtils.isEmpty(loginUser.getUser().getRoles())) + { + return false; + } + for (SysRole sysRole : loginUser.getUser().getRoles()) + { + String roleKey = sysRole.getRoleKey(); + if (SUPER_ADMIN.equals(roleKey) || roleKey.equals(StringUtils.trim(role))) + { + return true; + } + } + return false; + } + + /** + * 验证用户是否不具备某角色,与 isRole逻辑相反。 + * + * @param role 角色名称 + * @return 用户是否不具备某角色 + */ + public boolean lacksRole(String role) + { + return hasRole(role) != true; + } + + /** + * 验证用户是否具有以下任意一个角色 + * + * @param roles 以 ROLE_NAMES_DELIMETER 为分隔符的角色列表 + * @return 用户是否具有以下任意一个角色 + */ + public boolean hasAnyRoles(String roles) + { + if (StringUtils.isEmpty(roles)) + { + return false; + } + LoginUser loginUser = SecurityUtils.getLoginUser(); + if (StringUtils.isNull(loginUser) || CollectionUtils.isEmpty(loginUser.getUser().getRoles())) + { + return false; + } + for (String role : roles.split(ROLE_DELIMETER)) + { + if (hasRole(role)) + { + return true; + } + } + return false; + } + + /** + * 判断是否包含权限 + * + * @param permissions 权限列表 + * @param permission 权限字符串 + * @return 用户是否具备某权限 + */ + private boolean hasPermissions(Set permissions, String permission) + { + return permissions.contains(ALL_PERMISSION) || permissions.contains(StringUtils.trim(permission)); + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysLoginService.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysLoginService.java new file mode 100644 index 0000000..bb8970b --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysLoginService.java @@ -0,0 +1,186 @@ +package net.hzvtc.framework.web.service; + +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.core.domain.AjaxResult; +import net.hzvtc.common.core.domain.entity.SysUser; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.core.redis.RedisCache; +import net.hzvtc.common.exception.ServiceException; +import net.hzvtc.common.exception.user.CaptchaException; +import net.hzvtc.common.exception.user.CaptchaExpireException; +import net.hzvtc.common.exception.user.UserPasswordNotMatchException; +import net.hzvtc.common.utils.DateUtils; +import net.hzvtc.common.utils.MessageUtils; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.ip.IpUtils; +import net.hzvtc.email.service.EmailService; +import net.hzvtc.framework.manager.AsyncManager; +import net.hzvtc.framework.manager.factory.AsyncFactory; +import net.hzvtc.system.service.SysConfigService; +import net.hzvtc.system.service.SysUserService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Random; + +/** + * 登录校验方法 + * + * @author admin + */ +@Component +public class SysLoginService { + @Autowired + private TokenService tokenService; + + @Resource + private AuthenticationManager authenticationManager; + + @Autowired + private RedisCache redisCache; + + @Autowired + private SysUserService userService; + + @Autowired + private SysConfigService configService; + + @Autowired + private EmailService emailService; + + /** + * 登录验证 + * + * @param username 用户名 + * @param password 密码 + * @param code 验证码 + * @param uuid 唯一标识 + * @return 结果 + */ + public String login(String username, String password, String code, String uuid, int loginType) { + //sso 指定用户登录 +// if (loginType == 3) { +// username = "ssoLoginUser"; +// password = "ssoLoginUser_123"; +// } else { +// boolean captchaOnOff = configService.selectCaptchaOnOff(); +// // 验证码开关 +// if (captchaOnOff && loginType == 0) { +// validateCaptcha(username, code, uuid); +// } +// } + boolean captchaOnOff = configService.selectCaptchaOnOff(); + // 验证码开关 + if (captchaOnOff && (loginType == 0 || loginType == 2)) { + validateCaptcha(username, code, uuid); + } + // 用户验证 + Authentication authentication = null; + try { + // 该方法会去调用UserDetailsServiceImpl.loadUserByUsername + authentication = authenticationManager.authenticate(new UsernamePasswordAuthenticationToken(username, password)); + } catch (Exception e) { + if (e instanceof BadCredentialsException) { + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("user.password.not.match"))); + throw new UserPasswordNotMatchException(); + } else { + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, e.getMessage())); + throw new ServiceException(e.getMessage()); + } + } + LoginUser loginUser = (LoginUser) authentication.getPrincipal(); +// if (loginType == 0) { +// if (loginUser.getPortalLogin() != 1) { +// AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("no.portal.login"))); +// throw new UserNoLoginPermissionException(); +// } +// } else if (loginType == 1) { +// if (loginUser.getWebLogin() != 1) { +// AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("no.web.login"))); +// throw new UserNoLoginPermissionException(); +// } +// } + + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_SUCCESS, MessageUtils.message("user.login.success"))); + + recordLoginInfo(loginUser.getUserId()); + // 生成token + return tokenService.createToken(loginUser); + } + + /** + * 校验验证码 + * + * @param username 用户名 + * @param code 验证码 + * @param uuid 唯一标识 + * @return 结果 + */ + public void validateCaptcha(String username, String code, String uuid) { + String verifyKey = Constants.CAPTCHA_CODE_KEY + uuid; + String captcha = redisCache.getCacheObject(verifyKey); + redisCache.deleteObject(verifyKey); + if (captcha == null) { + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("user.jcaptcha.expire"))); + throw new CaptchaExpireException(); + } + if (!code.equalsIgnoreCase(captcha)) { + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("user.jcaptcha.error"))); + throw new CaptchaException(); + } + } + + /** + * 记录登录信息 + * + * @param userId 用户ID + */ + public void recordLoginInfo(Long userId) { + SysUser sysUser = new SysUser(); + sysUser.setUserId(userId); + sysUser.setLoginIp(IpUtils.getIpAddr(ServletUtils.getRequest())); + sysUser.setLoginDate(DateUtils.getNowDate()); + userService.updateUserProfile(sysUser); + } + + public AjaxResult resetPasswordByName(String loginId) { + SysUser user = userService.selectUserByUserName(loginId); + if (user == null) { + return AjaxResult.error(500, "帐号不存在"); + } else if (user.getEmail() == null || user.getEmail().isEmpty()) { + return AjaxResult.error(500, "此帐号电子邮箱为空"); + } + String passwd = genRandomNum(); + userService.resetUserPwd(loginId, SecurityUtils.encryptPassword(passwd)); + emailService.sendSimpleEmail("密码重置", "你的密码已经重置为:" + passwd + ",请及时修改!", user.getEmail()); + return AjaxResult.success("密码已重置!"); + } + + + private String genRandomNum() { + int maxNum = 36; + int i; + int count = 0; + char[] str = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}; + StringBuffer pwd = new StringBuffer(""); + Random r = new Random(); + while (count < 8) { + i = Math.abs(r.nextInt(maxNum)); + if (i >= 0 && i < str.length) { + pwd.append(str[i]); + count++; + } + } + return pwd.toString(); + } + + +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysPermissionService.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysPermissionService.java new file mode 100644 index 0000000..1006b36 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysPermissionService.java @@ -0,0 +1,62 @@ +package net.hzvtc.framework.web.service; + +import net.hzvtc.common.core.domain.entity.SysRole; +import net.hzvtc.common.core.domain.entity.SysUser; +import net.hzvtc.system.service.SysMenuService; +import net.hzvtc.system.service.SysRoleService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.Set; + +/** + * 用户权限处理 + * + * @author admin + */ +@Component +public class SysPermissionService { + @Autowired + private SysRoleService roleService; + + @Autowired + private SysMenuService menuService; + + /** + * 获取角色数据权限 + * + * @param user 用户信息 + * @return 角色权限信息 + */ + public Set getRolePermission(SysUser user) { + Set roles = new HashSet(); + // 管理员拥有所有权限 + if (user.isAdmin()) { + roles.add("admin"); + } else { + SysRole sysRole = new SysRole(); + sysRole.setUserId(user.getUserId()); + sysRole.setSysId(user.getSysId()); + roles.addAll(roleService.selectRolePermissionByUserId(sysRole)); + } + return roles; + } + + /** + * 获取菜单数据权限 + * + * @param user 用户信息 + * @return 菜单权限信息 + */ + public Set getMenuPermission(SysUser user) { + Set perms = new HashSet(); + // 管理员拥有所有权限 + if (user.isAdmin()) { + perms.add("*:*:*"); + } else { + perms.addAll(menuService.selectMenuPermsByUserId(user.getUserId())); + } + return perms; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysRegisterService.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysRegisterService.java new file mode 100644 index 0000000..7f38abe --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/SysRegisterService.java @@ -0,0 +1,95 @@ +package net.hzvtc.framework.web.service; + +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.constant.UserConstants; +import net.hzvtc.common.core.domain.entity.SysUser; +import net.hzvtc.common.core.domain.model.RegisterBody; +import net.hzvtc.common.core.redis.RedisCache; +import net.hzvtc.common.exception.user.CaptchaException; +import net.hzvtc.common.exception.user.CaptchaExpireException; +import net.hzvtc.common.utils.MessageUtils; +import net.hzvtc.common.utils.SecurityUtils; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.framework.manager.AsyncManager; +import net.hzvtc.framework.manager.factory.AsyncFactory; +import net.hzvtc.system.service.SysConfigService; +import net.hzvtc.system.service.SysUserService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 注册校验方法 + * + * @author admin + */ +@Component +public class SysRegisterService { + @Autowired + private SysUserService userService; + + @Autowired + private SysConfigService configService; + + @Autowired + private RedisCache redisCache; + + /** + * 注册 + */ + public String register(RegisterBody registerBody) { + String msg = "", username = registerBody.getUsername(), password = registerBody.getPassword(); + + boolean captchaOnOff = configService.selectCaptchaOnOff(); + // 验证码开关 + if (captchaOnOff) { + validateCaptcha(username, registerBody.getCode(), registerBody.getUuid()); + } + + if (StringUtils.isEmpty(username)) { + msg = "用户名不能为空"; + } else if (StringUtils.isEmpty(password)) { + msg = "用户密码不能为空"; + } else if (username.length() < UserConstants.USERNAME_MIN_LENGTH + || username.length() > UserConstants.USERNAME_MAX_LENGTH) { + msg = "账户长度必须在2到20个字符之间"; + } else if (password.length() < UserConstants.PASSWORD_MIN_LENGTH + || password.length() > UserConstants.PASSWORD_MAX_LENGTH) { + msg = "密码长度必须在5到20个字符之间"; + } else if (UserConstants.NOT_UNIQUE.equals(userService.checkUserNameUnique(username))) { + msg = "保存用户'" + username + "'失败,注册账号已存在"; + } else { + SysUser sysUser = new SysUser(); + sysUser.setUserName(username); + sysUser.setNickName(username); + sysUser.setPassword(SecurityUtils.encryptPassword(registerBody.getPassword())); + boolean regFlag = userService.registerUser(sysUser); + if (!regFlag) { + msg = "注册失败,请联系系统管理人员"; + } else { + AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.REGISTER, + MessageUtils.message("user.register.success"))); + } + } + return msg; + } + + /** + * 校验验证码 + * + * @param username 用户名 + * @param code 验证码 + * @param uuid 唯一标识 + * @return 结果 + */ + public void validateCaptcha(String username, String code, String uuid) { + String verifyKey = Constants.CAPTCHA_CODE_KEY + uuid; + String captcha = redisCache.getCacheObject(verifyKey); + redisCache.deleteObject(verifyKey); + if (captcha == null) { + throw new CaptchaExpireException(); + } + if (!code.equalsIgnoreCase(captcha)) { + throw new CaptchaException(); + } + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/TokenService.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/TokenService.java new file mode 100644 index 0000000..0073cca --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/TokenService.java @@ -0,0 +1,208 @@ +package net.hzvtc.framework.web.service; + +import eu.bitwalker.useragentutils.UserAgent; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.security.Keys; +import net.hzvtc.common.constant.Constants; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.core.redis.RedisCache; +import net.hzvtc.common.utils.ServletUtils; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.common.utils.ip.AddressUtils; +import net.hzvtc.common.utils.ip.IpUtils; +import net.hzvtc.common.utils.uuid.IdUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.crypto.SecretKey; +import javax.servlet.http.HttpServletRequest; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * token验证处理 + * + * @author admin + */ +@Component +public class TokenService { + protected static final long MILLIS_SECOND = 1000; + protected static final long MILLIS_MINUTE = 60 * MILLIS_SECOND; + private static final Long MILLIS_MINUTE_TEN = 20 * 60 * 1000L; + // 令牌自定义标识 + @Value("${token.header}") + private String header; + // 令牌秘钥 + @Value("${token.secret}") + private String secret; + // 令牌有效期(默认30分钟) + @Value("${token.expireTime}") + private int expireTime; + @Autowired + private RedisCache redisCache; + + /** + * 获取用户身份信息 + * + * @return 用户信息 + */ + public LoginUser getLoginUser(HttpServletRequest request) { + // 获取请求携带的令牌 + String token = getToken(request); + if (StringUtils.isNotEmpty(token)) { + try { + Claims claims = parseToken(token); + // 解析对应的权限以及用户信息 + String uuid = (String) claims.get(Constants.LOGIN_USER_KEY); + String userKey = getTokenKey(uuid); + LoginUser user = redisCache.getCacheObject(userKey); + return user; + } catch (Exception e) { + } + } + return null; + } + + /** + * 设置用户身份信息 + */ + public void setLoginUser(LoginUser loginUser) { + if (StringUtils.isNotNull(loginUser) && StringUtils.isNotEmpty(loginUser.getToken())) { + refreshToken(loginUser); + } + } + + /** + * 删除用户身份信息 + */ + public void delLoginUser(String token) { + if (StringUtils.isNotEmpty(token)) { + String userKey = getTokenKey(token); + redisCache.deleteObject(userKey); + } + } + + /** + * 创建令牌 + * + * @param loginUser 用户信息 + * @return 令牌 + */ + public String createToken(LoginUser loginUser) { + String token = IdUtils.fastUUID(); + loginUser.setToken(token); + setUserAgent(loginUser); + refreshToken(loginUser); + + Map claims = new HashMap<>(); + claims.put(Constants.LOGIN_USER_KEY, token); + return createToken(claims); + } + + /** + * 验证令牌有效期,相差不足20分钟,自动刷新缓存 + * + * @param loginUser + * @return 令牌 + */ + public void verifyToken(LoginUser loginUser) { + long expireTime = loginUser.getExpireTime(); + long currentTime = System.currentTimeMillis(); + if (expireTime - currentTime <= MILLIS_MINUTE_TEN) { + refreshToken(loginUser); + } + } + + /** + * 刷新令牌有效期 + * + * @param loginUser 登录信息 + */ + public void refreshToken(LoginUser loginUser) { + loginUser.setLoginTime(System.currentTimeMillis()); + loginUser.setExpireTime(loginUser.getLoginTime() + expireTime * MILLIS_MINUTE); + // 根据uuid将loginUser缓存 + String userKey = getTokenKey(loginUser.getToken()); + redisCache.setCacheObject(userKey, loginUser, expireTime, TimeUnit.MINUTES); + } + + /** + * 设置用户代理信息 + * + * @param loginUser 登录信息 + */ + public void setUserAgent(LoginUser loginUser) { + UserAgent userAgent = UserAgent.parseUserAgentString(ServletUtils.getRequest().getHeader("User-Agent")); + String ip = IpUtils.getIpAddr(ServletUtils.getRequest()); + loginUser.setIpaddr(ip); + loginUser.setLoginLocation(AddressUtils.getRealAddressByIP(ip)); + loginUser.setBrowser(userAgent.getBrowser().getName()); + loginUser.setOs(userAgent.getOperatingSystem().getName()); + } + + /** + * 从数据声明生成令牌 + * + * @param claims 数据声明 + * @return 令牌 + */ + private String createToken(Map claims) { +// String token = Jwts.builder() +// .setClaims(claims) +// .signWith(SignatureAlgorithm.HS512, secret).compact(); + SecretKey key = Keys.hmacShaKeyFor(secret.getBytes()); + String token = Jwts.builder().claims(claims).signWith(key, Jwts.SIG.HS512).compact(); +// signWith(SignatureAlgorithm.HS512, secret).compact(); + return token; + } + + /** + * 从令牌中获取数据声明 + * + * @param token 令牌 + * @return 数据声明 + */ + private Claims parseToken(String token) { + SecretKey key = Keys.hmacShaKeyFor(secret.getBytes()); + return Jwts.parser() + .verifyWith(key) + .build() + .parseSignedClaims(token).getPayload(); +// return Jwts.parser() +// .setSigningKey(secret) +// .parseClaimsJws(token) +// .getBody(); + } + + /** + * 从令牌中获取用户名 + * + * @param token 令牌 + * @return 用户名 + */ + public String getUsernameFromToken(String token) { + Claims claims = parseToken(token); + return claims.getSubject(); + } + + /** + * 获取请求token + * + * @param request + * @return token + */ + private String getToken(HttpServletRequest request) { + String token = request.getHeader(header); + if (StringUtils.isNotEmpty(token) && token.startsWith(Constants.TOKEN_PREFIX)) { + token = token.replace(Constants.TOKEN_PREFIX, ""); + } + return token; + } + + private String getTokenKey(String uuid) { + return Constants.LOGIN_TOKEN_KEY + uuid; + } +} diff --git a/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/UserDetailsServiceImpl.java b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/UserDetailsServiceImpl.java new file mode 100644 index 0000000..582e824 --- /dev/null +++ b/ai-xuebao-framework/src/main/java/net/hzvtc/framework/web/service/UserDetailsServiceImpl.java @@ -0,0 +1,75 @@ +package net.hzvtc.framework.web.service; + +import com.alibaba.fastjson.JSONObject; +import net.hzvtc.area.service.ProtectedAreaService; +import net.hzvtc.common.core.domain.entity.SysUser; +import net.hzvtc.common.core.domain.model.LoginUser; +import net.hzvtc.common.enums.UserStatus; +import net.hzvtc.common.exception.ServiceException; +import net.hzvtc.common.utils.StringUtils; +import net.hzvtc.device.service.DeviceInfoService; +import net.hzvtc.system.service.SysRoleService; +import net.hzvtc.system.service.SysUserService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.core.userdetails.UserDetailsService; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * 用户验证处理 + * + * @author admin + */ +@Service +public class UserDetailsServiceImpl implements UserDetailsService { + private static final Logger log = LoggerFactory.getLogger(UserDetailsServiceImpl.class); + + @Autowired + private SysUserService userService; + + @Autowired + private SysPermissionService permissionService; + + @Autowired + private SysRoleService sysRoleService; + + @Autowired + private ProtectedAreaService protectedAreaService; + + @Autowired + private DeviceInfoService deviceInfoService; + + @Override + public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { + SysUser user = userService.selectUserByUserName(username); + if (StringUtils.isNull(user)) { + log.info("登录用户:{} 不存在.", username); + throw new ServiceException("登录用户:" + username + " 不存在"); + } else if (UserStatus.DELETED.getCode().equals(user.getDelFlag())) { + log.info("登录用户:{} 已被删除.", username); + throw new ServiceException("对不起,您的账号:" + username + " 已被删除"); + } else if (UserStatus.DISABLE.getCode().equals(user.getStatus())) { + log.info("登录用户:{} 已被停用.", username); + throw new ServiceException("对不起,您的账号:" + username + " 已停用"); + } + return createLoginUser(user); + } + + public UserDetails createLoginUser(SysUser user) { +// List areaList = protectedAreaService.getAreaListByUserId(user.getUserId()); +// Long userId = null; +// if(user.getDeptId()==1L){ +// userId = 0L; +// }else{ +// userId = user.getUserId(); +// } +// List deviceList = deviceInfoService.getDeviceInfoListByUserId(userId); + return new LoginUser(user.getUserId(), user.getDeptId(), user, permissionService.getMenuPermission(user), + user.getWebLogin(), user.getPortalLogin()); + } +}