解决.ToPivotTable(it => it.day, it =>new{ it.name}, it => it.Sum(x => x.count))的几个问题

1 it.name 为空值时抛出异常索引超出界值异常
2 当it.name 的值中 有"T0 = T1 = T2"类似情况是会截断字符串,显示为"T0"
3 提升转化效率,现在所花时间只需要之前的1/4左右
This commit is contained in:
YHS 2024-01-05 11:06:06 +08:00
parent 305c36924d
commit 079abab4e5
2 changed files with 165 additions and 40 deletions

View File

@ -0,0 +1,126 @@
#region using
using OrmTest;
using SqlSugar;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
#endregion
namespace MySqlTest
{
public class _a9_ToPivot
{
public static void Init()
{
var db = DbHelper.GetNewDb();
db.CodeFirst.SetStringDefaultLength(200).InitTables(typeof(RowToColumn));
Clean();
InitData();
Test5();
Test6();
}
public static void InitData()
{
var db = DbHelper.GetNewDb();
var ls = new List<RowToColumn>();
// 创建 Stopwatch 对象并开始计时
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 1; i <= 100 * 10000; i++)
{
ls.Add(new RowToColumn()
{
Id = Guid.NewGuid(),
Code = "A" + i % 1000,
Date = new DateTime(2023, i % 11 + 1, 1),
Date2 = new DateTime(2023, i % 11 + 1, i % 28 + 1, i % 24, i % 60, i % 60),
Val = i % 10,
T1 = i % 3 == 0 ? null : "T0",
T2 = i % 3 == 1 ? null : 3
}); ;
if (i % 100000 == 0)
{
db.Fastest<RowToColumn>().PageSize(100000).BulkCopy(ls);
ls.Clear();
}
}
Task.Run(() =>
{
});
// 结束计时并获取经过的时间
stopwatch.Stop();
TimeSpan elapsedTime = stopwatch.Elapsed;
Console.WriteLine("T0:" + elapsedTime);
}
public static void Clean()
{
var db = DbHelper.GetNewDb();
db.DbMaintenance.TruncateTable<RowToColumn>();
}
public static void Test5()
{
var db = DbHelper.GetNewDb();
// 创建 Stopwatch 对象并开始计时
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var s = new DateTime(2023, 2, 1);
var e = new DateTime(2023, 6, 1).AddMilliseconds(-1);
var ls = db.Queryable<RowToColumn>()
.Where(x => x.Date >= s && x.Date <= e)
.GroupBy(x => new { x.Code, x.Date, x.Val, x.T1, x.T2 })
.Select(x => new { x.Code, x.Date, x.T1, x.T2, Val = SqlFunc.AggregateSum(x.Val) })//求和可以自动处理空值
.ToPivotTable(it => it.Code, it => new { it.Date, it.T1, it.T2 }, it => it.Any() ? it.Sum(x => x.Val) : 0);
// 结束计时并获取经过的时间
stopwatch.Stop();
TimeSpan elapsedTime = stopwatch.Elapsed;
Console.WriteLine("T5:" + elapsedTime);
}
public static void Test6()
{
var db = DbHelper.GetNewDb();
// 创建 Stopwatch 对象并开始计时
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var s = new DateTime(2023, 2, 1);
var e = new DateTime(2023, 6, 1).AddMilliseconds(-1);
var ls = db.Queryable<RowToColumn>()
.Where(x => x.Date >= s && x.Date <= e)
.GroupBy(x => new { x.Code, x.Date, x.Val, x.T1, x.T2 })
.Select(x => new { x.Code, x.Date, x.T1, x.T2, Val = SqlFunc.AggregateSum(x.Val) })//求和可以自动处理空值
.ToPivotList(it => it.Code, it => new { it.Date, it.T1, it.T2 }, it => it.Any() ? it.Sum(x => x.Val) : 0);
// 结束计时并获取经过的时间
stopwatch.Stop();
TimeSpan elapsedTime = stopwatch.Elapsed;
Console.WriteLine("T000:" + elapsedTime);
}
}
[SugarTable("row_2_column")]
public class RowToColumn
{
[SugarColumn(IsPrimaryKey = true)]
public Guid Id { get; set; }
public DateTime Date { get; set; }
public DateTime Date2 { get; set; }
public string Code { get; set; }
public decimal? Val { get; set; }
[SugarColumn(ColumnName = "t_1", IsNullable = true)]
public string T1 { get; set; }
[SugarColumn(ColumnName = "t_2", IsNullable = true)]
public int? T2 { get; set; }
}
}

View File

@ -2,13 +2,15 @@
using System.Collections.Generic;
using System.Data;
using System.Dynamic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace SqlSugar
namespace SqlSugar
{
internal static class DataTableExtensions
{
@ -19,12 +21,13 @@ namespace SqlSugar
Func<IEnumerable<T>, TData> dataSelector)
{
DataTable table = new DataTable();
var rowName = "";
var rowName = new List<string>();
if (rowSelector.Body is MemberExpression)
rowName = ((MemberExpression)rowSelector.Body).Member.Name;
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
else
rowName =string.Join(UtilConstants.ReplaceKey, ((NewExpression)rowSelector.Body).Arguments.Select(it=>it as MemberExpression).Select(it=>it.Member.Name));
table.Columns.Add(new DataColumn(rowName));
rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name));
table.Columns.AddRange(rowName.Select(x => new DataColumn(x)).ToArray());
var columns = source.Select(columnSelector).Distinct();
foreach (var column in columns)
@ -45,36 +48,23 @@ namespace SqlSugar
{
var dataRow = table.NewRow();
var items = row.Values.Cast<object>().ToList();
items.Insert(0, row.Key);
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{
items.Insert(i, properties[i].GetValue(row.Key, null));
}
dataRow.ItemArray = items.ToArray();
table.Rows.Add(dataRow);
}
var firstName = table.Columns[0]?.ColumnName;
if (firstName.ObjToString().Contains(UtilConstants.ReplaceKey))
{
int i = 0;
foreach (var item in Regex.Split(firstName,UtilConstants.ReplaceKey))
{
i++;
table.Columns.Add(item);
table.Columns[item].SetOrdinal(i);
}
foreach (DataRow row in table.Rows)
{
var json =row[firstName];
var list = json.ToString().TrimStart('{', ' ').TrimEnd('}', ' ')
.Split(new[] { ", " }, StringSplitOptions.None)
.Select(it => it.Split(new[] { " = " }, StringSplitOptions.None)).ToList();
foreach (var item in Regex.Split(firstName, UtilConstants.ReplaceKey))
{
var x = list.First(it => it.First().Trim() == item.Trim());
row[item] =x[1] ;
}
}
table.Columns.Remove(firstName);
}
return table;
}
public static List<dynamic> ToPivotList<T, TColumn, TRow, TData>(
this IEnumerable<T> source,
Func<T, TColumn> columnSelector,
@ -82,17 +72,17 @@ namespace SqlSugar
Func<IEnumerable<T>, TData> dataSelector)
{
var arr = new List<object>();
var arr = new List<dynamic>();
var cols = new List<string>();
var rowName = "";
var rowName = new List<string>();
if (rowSelector.Body is MemberExpression)
rowName = ((MemberExpression)rowSelector.Body).Member.Name;
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
else
rowName = "Group_"+string.Join("_", ((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name));
rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name));
var columns = source.Select(columnSelector).Distinct();
cols = (new[] { rowName }).Concat(columns.Select(x => x?.ToString())).ToList();
cols = rowName.Concat(columns.Select(x => x?.ToString())).ToList();
var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new
@ -103,17 +93,26 @@ namespace SqlSugar
c => c,
r => columnSelector(r),
(c, columnGroup) => dataSelector(columnGroup))
}).ToList();
});
foreach (var row in rows)
{
var items = row.Values.Cast<object>().ToList();
items.Insert(0, row.Key);
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{
items.Insert(i, properties[i].GetValue(row.Key, null));
}
var obj = GetAnonymousObject(cols, items);
arr.Add(obj);
}
return arr.ToList();
return arr;
}
private static dynamic GetAnonymousObject(IEnumerable<string> columns, IEnumerable<object> values)
{