QRAPI/API/Wood.Service/Controllers/RecurringJobOutPageController.cs

using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using Omu.ValueInjecter;
using System.Linq.Expressions;
using System.Text.Json;
using System.Text.Json.Serialization;
using TaskManager.Contracts.Dtos;
using TaskManager.Controllers;
using TaskManager.Entity;
using TaskManager.EntityFramework;
using TaskManager.EntityFramework;
using TaskManager.EntityFramework.Repository;

namespace TaskManager.Controllers
{
    public class RecurringJobOutPageController<T, ToutputDetial> : RecurringJobBaseController where T : CherryReadBaseEntity, new() where ToutputDetial : CherryReadBaseEntityDto
    {


        protected readonly IRepository<T> _repository;
        public RecurringJobOutPageController(HttpClient httpClient, JobDbContext jobDbContext, LogController log,IRepository<T> repository) : base(httpClient, jobDbContext, log)
        {
            _repository = repository;
        }

        public async Task<List<ToutputDetial>> FetchAllDataAsync(string inputdate)
        {
            var allData = new List<ToutputDetial>();
            int totalItems = 0;
            int pageSize = 0;
            int currentPage = 1;
            string date = !string.IsNullOrEmpty(inputdate) ? inputdate : DateTime.Now.ToString("yyyy-MM-dd");


            // 首次请求获取总条数和分页信息
            PagedResponse<ToutputDetial> firstResponse = await GetPageAsync(new PAGE_DTO() { Date = date, IsForce = true });



            if (firstResponse == null || firstResponse.Code != 200)
            {
                await _logger.AddError("首次请求失败，无法获取分页信息。", TaskName);
                return allData;
            }
            if (firstResponse.Data.Total == "0")
            {
                await _logger.AddError("首次请求失败，Total为0是否已经全部读取过。", TaskName);
                return allData;

            }

            var readedcount = _jobDbContext.Set<T>().Where(p => p.RequestDate == inputdate).Count();
            if (readedcount != int.Parse(firstResponse.Data.Total))//记录数不相等
            {
                var ids = _jobDbContext.Set<T>().Where(p => p.RequestDate == inputdate).Select(p => p.Id).ToList();//已经同步的ID
                totalItems = int.Parse(firstResponse.Data.Total);
                pageSize = int.Parse(firstResponse.Data.PageSize);
                List<T> pagefirstList = new List<T>();
                if (readedcount > 0)
                {
                    var listrows = firstResponse.Data.Rows.Where(p => !ids.Contains(p.Id));
                    foreach (var itm in firstResponse.Data.Rows)
                    {
                        T entity = new T();
                        entity.InjectFrom(itm);
                        entity.CreationTime = DateTime.Now;
                        pagefirstList.Add(entity);
                        allData.Add(itm);
                    }
                }
                else
                {

                    foreach (var itm in firstResponse.Data.Rows)
                    {
                        T entity = new T();
                        entity.InjectFrom(itm);
                        entity.CreationTime = DateTime.Now;
                        pagefirstList.Add(entity);
                        allData.Add(itm);
                    }
                }

                _jobDbContext.BulkInsert(pagefirstList);

                //Console.WriteLine($"总记录数: {totalItems}, 每页大小: {pageSize}");

                // 计算总页数
                int totalPages = (int)Math.Ceiling((double)totalItems / pageSize);
                //Console.WriteLine($"总共需要请求 {totalPages} 页数据");

                // 循环请求剩余页面
                for (currentPage = 2; currentPage <= totalPages; currentPage++)
                {
                    PAGE_DTO pageinput = new PAGE_DTO() { Date = date, PageNum = currentPage, IsForce = true };
                    Console.WriteLine($"正在请求第 {currentPage} 页...");
                    PagedResponse<ToutputDetial> pageResponse = await GetPageAsync(pageinput);
                    if (pageResponse?.Data.Rows != null && pageResponse.Data.Rows.Count > 0)
                    {
                        List<T> pageList = new List<T>();
                        //foreach (var itm in pageResponse.Data.Rows)
                        //{
                        //    T entity = new T();
                        //    entity.InjectFrom(itm);
                        //    entity.CreationTime = DateTime.Now;
                        //    pageList.Add(entity);
                        //    allData.Add(itm);
                        //}
                        if (readedcount > 0)
                        {
                            var listrows = pageResponse.Data.Rows.Where(p => !ids.Contains(p.Id));
                            foreach (var itm in pageResponse.Data.Rows)
                            {
                                T entity = new T();
                                entity.InjectFrom(itm);
                                entity.CreationTime = DateTime.Now;
                                pageList.Add(entity);
                                allData.Add(itm);
                            }
                        }
                        else
                        {

                            foreach (var itm in pageResponse.Data.Rows)
                            {
                                T entity = new T();
                                entity.InjectFrom(itm);
                                entity.CreationTime = DateTime.Now;
                                pageList.Add(entity);
                                allData.Add(itm);
                            }
                        }




                        _jobDbContext.BulkInsert(pageList);
                        await _logger.AddInfo($"成功获取 {pageResponse.Data.Rows.Count} 条记录", TaskName);
                    }
                    else
                    {
                        await _logger.AddError($"第 {currentPage} 页未返回数据", TaskName);

                    }

                    // 简单的请求间隔，避免过于频繁
                    await Task.Delay(200);
                }


                await _logger.AddInfo($"所有数据获取完成，总共获取了 {allData.Count} 条记录", TaskName);
            }


            return allData;
        }

        private async Task<PagedResponse<ToutputDetial>> GetPageAsync(PAGE_DTO input)
        {
            try
            {
                var inputjson = JsonSerializer.Serialize(input,
                   new JsonSerializerOptions
                   {
                       PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
                       WriteIndented = true // 可选，用于格式化输出
                   }
                   );
                inputjson = RemoveWhitespace(inputjson);
                var content = await Post(Url, Path, inputjson);
                if (!string.IsNullOrEmpty(content))
                {

                    var options = new JsonSerializerOptions
                    {
                        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
                        Converters = {
                         new JsonStringEnumConverter(), // 枚举转换
                         new CustomDateTimeConverter("yyyy-MM-dd HH:mm:ss") // 日期转换
    }
                    };
                    return JsonSerializer.Deserialize<PagedResponse<ToutputDetial>>(content, options);
                }
                else
                {
                    await _logger.AddError($"调用接口无返回值{Url}", TaskName);
                    return null;
                }
            }
            catch (Exception ex)
            {
                await _logger.AddError($"调用接口无返回值错误{ex.Message}", TaskName);
                return null;
            }

        }
        [HttpGet("Test")]
        public async Task TestAsync(string url, string path, string taskName, string inputdate)
        {
            Url = url;
            Path = path;
            TaskName = taskName;
            await FetchAllDataAsync(inputdate);
        }
        protected override async Task DoExecutingAsync(string url, string path, string takName)
        {
            Url = url;
            Path = path;
            TaskName = takName;
            await FetchAllDataAsync(string.Empty);

        }
        [HttpGet]
        public async Task<ActionResult<IEnumerable<T>>> GetAll()
        {
            return await _repository.GetAllAsync() as List<T>;
        }

        [HttpGet("{id}")]
        public async Task<ActionResult<T>> GetById(int id)
        {
            var entity = await _repository.GetByIdAsync(id);
            if (entity == null) return NotFound();
            return entity;
        }

        [HttpPost]
        public async Task<ActionResult<T>> Create(T entity)
        {
            var createdEntity = await _repository.AddAsync(entity);
            return CreatedAtAction(nameof(GetById), new { id = createdEntity.Id }, createdEntity);
        }

        [HttpPut("{id}")]
        public async Task<IActionResult> Update(int id, T entity)
        {
            if (id != entity.UId) return BadRequest();

            await _repository.UpdateAsync(entity);
            return NoContent();
        }

        [HttpDelete("{id}")]
        public async Task<IActionResult> Delete(int id)
        {
            await _repository.DeleteAsync(id);
            return NoContent();
        }
        [HttpGet]
        public async Task<ActionResult<PagedResult<T>>> GetPaged(
        [FromQuery] int pageNumber = 1,
        [FromQuery] int pageSize = 10,
        [FromQuery] string sortBy = "",
        [FromQuery] bool isAscending = true,
        [FromQuery] Dictionary<string, string> filters = null)
        {
            var pagingParams = new PagingParams
            {
                PageNumber = pageNumber,
                PageSize = pageSize,
                SortBy = sortBy,
                IsAscending = isAscending,
                Filters = filters
            };

            // 可以在这里构建表达式树过滤条件
            Expression<Func<T, bool>> filter = null;

            var pagedResult = await _repository.GetPagedAsync(filter, pagingParams);
            return Ok(pagedResult);
        }
        //GET /api/products?pageNumber=1&pageSize=10&filters[name]=Phone&filters[price]=999








    }
    public class CustomDateTimeConverter : JsonConverter<DateTime>
    {
        private readonly string _format;

        public CustomDateTimeConverter(string format)
        {
            _format = format;
        }

        public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        {
            return DateTime.ParseExact(reader.GetString(), _format, null);
        }

        public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options)
        {
            writer.WriteStringValue(value.ToString(_format));
        }
    }
}