샘플 데이터와 원하는 결과물을 올려주세요.

WITH T ( CATEGORY ,	DATA ) AS (
SELECT 'A','데이터1' FROM DUAL UNION ALL
SELECT 'A','데이터2' FROM DUAL UNION ALL
SELECT 'A','데이터3' FROM DUAL UNION ALL
SELECT 'A','데이터4' FROM DUAL UNION ALL
SELECT 'A','데이터5' FROM DUAL UNION ALL
SELECT 'A','데이터6' FROM DUAL UNION ALL
SELECT 'A','데이터7' FROM DUAL UNION ALL
SELECT 'B','데이터8' FROM DUAL UNION ALL
SELECT 'B','데이터9' FROM DUAL UNION ALL
SELECT 'B','데이터10' FROM DUAL UNION ALL
SELECT 'B','데이터11' FROM DUAL UNION ALL
SELECT 'B','데이터12' FROM DUAL UNION ALL
SELECT 'B','데이터13' FROM DUAL 
)

SELECT CATEGORY ,	DATA
  FROM (SELECT CATEGORY ,	DATA
             , ROW_NUMBER() OVER(PARTITION BY CATEGORY ORDER BY ROWNUM ) RN
          FROM T 
       )
 WHERE RN <= 3 
 ORDER BY CATEGORY , RN

SELECT category
     , data
  FROM (SELECT category
             , data
             , @rn := CASE WHEN @cd = category THEN @rn + 1 ELSE 1 END rn
             , @cd := category
          FROM (SELECT * FROM t ORDER BY category, 정렬기준) a
             , (SELECT @cd := '', @rn := 0) b
        ) a
 WHERE rn <= 3
;

잘못 적용하신 듯 하네요.