|
|
|
@ -324,6 +324,101 @@ namespace TaskManager.Controllers |
|
|
|
return allData; |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<List<ToutputDetial>> CustomFetchAllDataAsync(string inputdate) |
|
|
|
{ |
|
|
|
var allData = new List<ToutputDetial>(); |
|
|
|
int totalItems = 0; |
|
|
|
int pageSize = 0; |
|
|
|
int currentPage = 1; |
|
|
|
string date = !string.IsNullOrEmpty(inputdate) ? inputdate : DateTime.Now.ToString("yyyy-MM-dd"); |
|
|
|
var taskId = Guid.NewGuid(); |
|
|
|
var version = date.Replace("-", ""); |
|
|
|
var readedcount = _jobDbContext.Set<T>().Where(p => p.RequestDate == inputdate).Count(); |
|
|
|
|
|
|
|
|
|
|
|
PagedResponse<ToutputDetial> firstResponse = await GetPageAsync(new PAGE_DTO() { Date = date, PageSize = CPageSize, IsForce = true }, taskId, version); |
|
|
|
if (firstResponse == null || firstResponse.Code != 200) |
|
|
|
{ |
|
|
|
await _logger.AddInfo("首次请求失败，无法获取分页信息。", TaskName, taskId, version); |
|
|
|
return allData; |
|
|
|
} |
|
|
|
if (firstResponse.Data.Total == "0") |
|
|
|
{ |
|
|
|
await _logger.AddInfo("首次请求失败，Total为0已经全部读取过。", TaskName, taskId, version); |
|
|
|
return allData; |
|
|
|
|
|
|
|
} |
|
|
|
if (readedcount != int.Parse(firstResponse.Data.Total))//记录数不相等
|
|
|
|
{ |
|
|
|
var ids = _jobDbContext.Set<T>().Where(p => p.RequestDate == inputdate).Select(p => p.Id).ToList();//已经同步的ID
|
|
|
|
totalItems = int.Parse(firstResponse.Data.Total); |
|
|
|
pageSize = int.Parse(firstResponse.Data.PageSize); |
|
|
|
List<T> pagefirstList = new List<T>(); |
|
|
|
|
|
|
|
foreach (var itm in firstResponse.Data.Rows) |
|
|
|
{ |
|
|
|
T entity = new T(); |
|
|
|
entity.InjectFrom(itm); |
|
|
|
entity.CreationTime = DateTime.Now; |
|
|
|
pagefirstList.Add(entity); |
|
|
|
entity.RequestDate = date; |
|
|
|
allData.Add(itm); |
|
|
|
} |
|
|
|
if (pagefirstList.Any()) |
|
|
|
{ |
|
|
|
await InsertDataAsync(pagefirstList); |
|
|
|
} |
|
|
|
//Console.WriteLine($"总记录数: {totalItems}, 每页大小: {pageSize}");
|
|
|
|
// 计算总页数
|
|
|
|
int totalPages = (int)Math.Ceiling((double)totalItems / pageSize); |
|
|
|
//Console.WriteLine($"总共需要请求 {totalPages} 页数据");
|
|
|
|
|
|
|
|
// 循环请求剩余页面
|
|
|
|
for (currentPage = 2; currentPage <= totalPages; currentPage++) |
|
|
|
{ |
|
|
|
PAGE_DTO pageinput = new PAGE_DTO() { Date = date, PageNum = currentPage, PageSize = CPageSize, IsForce = true }; |
|
|
|
//Console.WriteLine($"正在请求第 {currentPage} 页...");
|
|
|
|
PagedResponse<ToutputDetial> pageResponse = await GetPageAsync(pageinput, taskId, version); |
|
|
|
if (pageResponse?.Data.Rows != null && pageResponse.Data.Rows.Count > 0) |
|
|
|
{ |
|
|
|
List<T> pageList = new List<T>(); |
|
|
|
foreach (var itm in pageResponse.Data.Rows) |
|
|
|
{ |
|
|
|
T entity = new T(); |
|
|
|
entity.InjectFrom(itm); |
|
|
|
entity.CreationTime = DateTime.Now; |
|
|
|
entity.RequestDate = date; |
|
|
|
pageList.Add(entity); |
|
|
|
allData.Add(itm); |
|
|
|
} |
|
|
|
if (pageList.Any()) |
|
|
|
{ |
|
|
|
await InsertDataAsync(pageList); |
|
|
|
} |
|
|
|
await _logger.AddSuccess($"成功获取 {pageResponse.Data.Rows.Count} 条记录", TaskName, taskId, version); |
|
|
|
} |
|
|
|
else |
|
|
|
{ |
|
|
|
await _logger.AddInfo($"第 {currentPage} 页未返回数据", TaskName, taskId, version); |
|
|
|
} |
|
|
|
// 简单的请求间隔，避免过于频繁
|
|
|
|
await Task.Delay(200); |
|
|
|
} |
|
|
|
await _logger.AddSuccess($"所有数据获取完成，总共获取了 {allData.Count} 条记录", TaskName, taskId, version); |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// 首次请求获取总条数和分页信息
|
|
|
|
|
|
|
|
return allData; |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private async Task<PagedResponse<ToutputDetial>> GetPageAsync(PAGE_DTO input,Guid taskId,string version) |
|
|
|
{ |
|
|
|
string inputjson = string.Empty; |
|
|
|
@ -385,7 +480,7 @@ namespace TaskManager.Controllers |
|
|
|
Path = first.Api; |
|
|
|
TaskName = taskName; |
|
|
|
CPageSize = first.PageSize; |
|
|
|
await FetchAllDataAsync(inputdate); |
|
|
|
await CustomFetchAllDataAsync(inputdate); |
|
|
|
} |
|
|
|
protected override async Task DoExecutingAsync(string url, string path, string takName, string client, int pagesize) |
|
|
|
{ |
|
|
|
|
