diff --git a/pkg/executor/historical_stats_test.go b/pkg/executor/historical_stats_test.go
index 65e54f20f5140..7aadeb34b986d 100644
--- a/pkg/executor/historical_stats_test.go
+++ b/pkg/executor/historical_stats_test.go
@@ -138,7 +138,7 @@ func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) {
 	tk.MustExec("delete from test.t where test.t.a = 1")
 	err = h.DumpStatsDeltaToKV(true)
 	require.NoError(t, err)
-	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d' order by create_time desc", tableInfo.Meta().ID)).Sort().Check(
+	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check(
 		testkit.Rows("40 20"))
 	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check(
 		testkit.Rows("40 20"))
@@ -147,7 +147,7 @@ func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) {
 	tk.MustExec("update test.t set test.t.b = 4 where test.t.a = 2")
 	err = h.DumpStatsDeltaToKV(true)
 	require.NoError(t, err)
-	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d' order by create_time desc", tableInfo.Meta().ID)).Sort().Check(
+	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check(
 		testkit.Rows("50 20"))
 	tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check(
 		testkit.Rows("50 20"))
diff --git a/pkg/executor/test/executor/executor_test.go b/pkg/executor/test/executor/executor_test.go
index 24dc4ef0f3f68..e3361749a8bc4 100644
--- a/pkg/executor/test/executor/executor_test.go
+++ b/pkg/executor/test/executor/executor_test.go
@@ -3175,6 +3175,14 @@ func TestIsPointGet(t *testing.T) {
 	}
 }
 
+func TestPointGetOrderby(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t (i int key)")
+	require.Equal(t, tk.ExecToErr("select * from t where i = 1 order by j limit 10;").Error(), "[planner:1054]Unknown column 'j' in 'order clause'")
+}
+
 func TestClusteredIndexIsPointGet(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)
diff --git a/pkg/planner/core/point_get_plan.go b/pkg/planner/core/point_get_plan.go
index 405885a320f27..a6c4da84cb7f5 100644
--- a/pkg/planner/core/point_get_plan.go
+++ b/pkg/planner/core/point_get_plan.go
@@ -1016,7 +1016,7 @@ func tryWhereIn2BatchPointGet(ctx sessionctx.Context, selStmt *ast.SelectStmt) *
 // 3. All the columns must be public and not generated.
 // 4. The condition is an access path that the range is a unique key.
 func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt, check bool) *PointGetPlan {
-	if selStmt.Having != nil {
+	if selStmt.Having != nil || selStmt.OrderBy != nil {
 		return nil
 	} else if selStmt.Limit != nil {
 		count, offset, err := extractLimitCountOffset(ctx, selStmt.Limit)
diff --git a/tests/integrationtest/r/explain_easy_stats.result b/tests/integrationtest/r/explain_easy_stats.result
index 1e508ac0300b9..1d375da7ac63c 100644
--- a/tests/integrationtest/r/explain_easy_stats.result
+++ b/tests/integrationtest/r/explain_easy_stats.result
@@ -178,7 +178,9 @@ id	estRows	task	access object	operator info
 Point_Get	1.00	root	table:index_prune, index:PRIMARY(a, b)	
 explain format = 'brief' select * from index_prune WHERE a = 1010010404050976781 AND b = 26467085526790 GROUP BY b ORDER BY a limit 1;
 id	estRows	task	access object	operator info
-Point_Get	1.00	root	table:index_prune, index:PRIMARY(a, b)	
+TopN	1.00	root		explain_easy_stats.index_prune.a, offset:0, count:1
+└─StreamAgg	1.00	root		group by:explain_easy_stats.index_prune.b, funcs:firstrow(explain_easy_stats.index_prune.a)->explain_easy_stats.index_prune.a, funcs:firstrow(explain_easy_stats.index_prune.b)->explain_easy_stats.index_prune.b, funcs:firstrow(explain_easy_stats.index_prune.c)->explain_easy_stats.index_prune.c
+  └─Point_Get	1.00	root	table:index_prune, index:PRIMARY(a, b)	
 drop table if exists t1, t2, t3, index_prune;
 set @@session.tidb_opt_insubq_to_join_and_agg=1;
 drop table if exists tbl;