read-csv-files(Scala)
Loading...

Read and Write CSV Files

%scala 
val diamonds = spark.read.format("csv")
  .option("header", "true")
  .option("inferSchema", "true")
  .load("/databricks-datasets/Rdatasets/data-001/csv/ggplot2/diamonds.csv")

display(diamonds)
10.23IdealESI261.5553263.953.982.43
20.21PremiumESI159.8613263.893.842.31
30.23GoodEVS156.9653274.054.072.31
40.29PremiumIVS262.4583344.24.232.63
50.31GoodJSI263.3583354.344.352.75
60.24Very GoodJVVS262.8573363.943.962.48
70.24Very GoodIVVS162.3573363.953.982.47
80.26Very GoodHSI161.9553374.074.112.53
90.22FairEVS265.1613373.873.782.49
100.23Very GoodHVS159.46133844.052.39
110.3GoodJSI164553394.254.282.73
120.23IdealJVS162.8563403.933.92.46
130.22PremiumFSI160.4613423.883.842.33
140.31IdealJSI262.2543444.354.372.71
150.2PremiumESI260.2623453.793.752.27
160.32PremiumEI160.9583454.384.422.68
170.3IdealISI262543484.314.342.68
180.3GoodJSI163.4543514.234.292.7
190.3GoodJSI163.8563514.234.262.71
200.3Very GoodJSI162.7593514.214.272.66
210.3GoodISI263.3563514.264.32.71
220.23Very GoodEVS263.8553523.853.922.48
230.23Very GoodHVS161573533.943.962.41
240.31Very GoodJSI159.4623534.394.432.62
250.31Very GoodJSI158.1623534.444.472.59
260.23Very GoodGVVS260.4583543.974.012.41
270.24PremiumIVS162.5573553.973.942.47
280.3Very GoodJVS262.2573574.284.32.67
290.23Very GoodDVS260.5613573.963.972.4
300.23Very GoodFVS160.9573573.963.992.42
310.23Very GoodFVS1605740244.032.41
320.23Very GoodFVS159.8574024.044.062.42
330.23Very GoodEVS160.7594023.974.012.42
340.23Very GoodEVS159.5584024.014.062.4
350.23Very GoodDVS161.9584023.923.962.44
360.23GoodFVS158.2594024.064.082.37
370.23GoodEVS164.1594023.833.852.46
380.31GoodHSI164544024.294.312.75
390.26Very GoodDVS260.8594034.134.162.52
400.33IdealISI261.8554034.494.512.78
410.33IdealISI261.2564034.494.52.75
420.33IdealJSI161.1564034.494.552.76
430.26GoodDVS265.2564033.994.022.61
440.26GoodDVS158.4634034.194.242.46
450.32GoodHSI263.1564034.344.372.75
460.29PremiumFSI162.4584034.244.262.65
470.32Very GoodHSI261.8554034.354.422.71
480.32GoodHSI263.8564034.364.382.79
490.25Very GoodEVS263.36040444.032.54
500.29Very GoodHSI260.7604044.334.372.64
510.24Very GoodFSI160.9614044.024.032.45
520.23IdealGVS161.9544043.933.952.44
530.32IdealISI160.9554044.454.482.72
540.22PremiumEVS261.6584043.933.892.41
550.22PremiumDVS259.3624043.913.882.31
560.3IdealISI261594054.34.332.63
570.3PremiumJSI259.3614054.434.382.61
580.3Very GoodISI162.6574054.254.282.67
590.3Very GoodISI163574054.284.322.71
600.3GoodISI163.2554054.254.292.7
610.35IdealIVS160.9575524.544.592.78
620.3PremiumDSI162.6595524.234.272.66
630.3IdealDSI162.5575524.294.322.69
640.3IdealDSI162.1565524.34.332.68
650.42PremiumISI261.5595524.784.842.96
660.28IdealGVVS261.4565534.194.222.58
670.32IdealIVVS16255.35534.394.422.73
680.31Very GoodGSI163.3575534.334.32.73
690.31PremiumGSI161.8585534.354.322.68
700.24PremiumEVVS160.7585534.014.032.44
710.24Very GoodDVVS161.5605533.9742.45
720.3Very GoodHSI163.1565544.294.272.7
730.3PremiumHSI162.9595544.284.242.68
740.3PremiumHSI162.5575544.294.252.67
750.3GoodHSI163.7575544.284.262.72
760.26Very GoodFVVS259.2605544.194.222.49
770.26Very GoodEVVS259.9585544.154.232.51
780.26Very GoodDVVS262.4545544.084.132.56
790.26Very GoodDVVS262.8605544.014.052.53
800.26Very GoodEVVS162.6595544.064.092.55
810.26Very GoodEVVS163.45955444.042.55
820.26Very GoodDVVS162.1605544.034.122.53
830.26IdealEVVS262.9585544.024.062.54
840.38IdealISI261.6565544.654.672.87
850.26GoodEVVS157.9605544.224.252.45
860.24PremiumGVVS162.3595543.953.922.45
870.24PremiumHVVS161.2585544.013.962.44
880.24PremiumHVVS160.8595544.0242.44
890.24PremiumHVVS260.7585544.074.042.46
900.32PremiumISI162.9585544.354.332.73
910.7IdealESI162.55727575.75.723.57
920.86FairESI255.16927576.456.333.52
930.7IdealGVS261.65627575.75.673.5
940.71Very GoodEVS262.45727595.685.733.56
950.78Very GoodGSI263.85627595.815.853.72
960.7GoodEVS257.55827595.855.93.38
970.7GoodFVS159.46227595.715.763.4
980.96FairFSI266.36227596.275.954.07
990.73Very GoodESI161.65927605.775.783.56
1000.8PremiumHSI161.55827605.975.933.66

Showing the first 1000 rows.

%scala

diamonds.printSchema
root |-- _c0: integer (nullable = true) |-- carat: double (nullable = true) |-- cut: string (nullable = true) |-- color: string (nullable = true) |-- clarity: string (nullable = true) |-- depth: double (nullable = true) |-- table: double (nullable = true) |-- price: integer (nullable = true) |-- x: double (nullable = true) |-- y: double (nullable = true) |-- z: double (nullable = true)
%r 
library(SparkR)

diamonds <- read.df("/databricks-datasets/Rdatasets/data-001/csv/ggplot2/diamonds.csv", source = "csv", header="true", inferSchema = "true")

display(diamonds)
10.23IdealESI261.5553263.953.982.43
20.21PremiumESI159.8613263.893.842.31
30.23GoodEVS156.9653274.054.072.31
40.29PremiumIVS262.4583344.24.232.63
50.31GoodJSI263.3583354.344.352.75
60.24Very GoodJVVS262.8573363.943.962.48
70.24Very GoodIVVS162.3573363.953.982.47
80.26Very GoodHSI161.9553374.074.112.53
90.22FairEVS265.1613373.873.782.49
100.23Very GoodHVS159.46133844.052.39
110.3GoodJSI164553394.254.282.73
120.23IdealJVS162.8563403.933.92.46
130.22PremiumFSI160.4613423.883.842.33
140.31IdealJSI262.2543444.354.372.71
150.2PremiumESI260.2623453.793.752.27
160.32PremiumEI160.9583454.384.422.68
170.3IdealISI262543484.314.342.68
180.3GoodJSI163.4543514.234.292.7
190.3GoodJSI163.8563514.234.262.71
200.3Very GoodJSI162.7593514.214.272.66
210.3GoodISI263.3563514.264.32.71
220.23Very GoodEVS263.8553523.853.922.48
230.23Very GoodHVS161573533.943.962.41
240.31Very GoodJSI159.4623534.394.432.62
250.31Very GoodJSI158.1623534.444.472.59
260.23Very GoodGVVS260.4583543.974.012.41
270.24PremiumIVS162.5573553.973.942.47
280.3Very GoodJVS262.2573574.284.32.67
290.23Very GoodDVS260.5613573.963.972.4
300.23Very GoodFVS160.9573573.963.992.42
310.23Very GoodFVS1605740244.032.41
320.23Very GoodFVS159.8574024.044.062.42
330.23Very GoodEVS160.7594023.974.012.42
340.23Very GoodEVS159.5584024.014.062.4
350.23Very GoodDVS161.9584023.923.962.44
360.23GoodFVS158.2594024.064.082.37
370.23GoodEVS164.1594023.833.852.46
380.31GoodHSI164544024.294.312.75
390.26Very GoodDVS260.8594034.134.162.52
400.33IdealISI261.8554034.494.512.78
410.33IdealISI261.2564034.494.52.75
420.33IdealJSI161.1564034.494.552.76
430.26GoodDVS265.2564033.994.022.61
440.26GoodDVS158.4634034.194.242.46
450.32GoodHSI263.1564034.344.372.75
460.29PremiumFSI162.4584034.244.262.65
470.32Very GoodHSI261.8554034.354.422.71
480.32GoodHSI263.8564034.364.382.79
490.25Very GoodEVS263.36040444.032.54
500.29Very GoodHSI260.7604044.334.372.64
510.24Very GoodFSI160.9614044.024.032.45
520.23IdealGVS161.9544043.933.952.44
530.32IdealISI160.9554044.454.482.72
540.22PremiumEVS261.6584043.933.892.41
550.22PremiumDVS259.3624043.913.882.31
560.3IdealISI261594054.34.332.63
570.3PremiumJSI259.3614054.434.382.61
580.3Very GoodISI162.6574054.254.282.67
590.3Very GoodISI163574054.284.322.71
600.3GoodISI163.2554054.254.292.7
610.35IdealIVS160.9575524.544.592.78
620.3PremiumDSI162.6595524.234.272.66
630.3IdealDSI162.5575524.294.322.69
640.3IdealDSI162.1565524.34.332.68
650.42PremiumISI261.5595524.784.842.96
660.28IdealGVVS261.4565534.194.222.58
670.32IdealIVVS16255.35534.394.422.73
680.31Very GoodGSI163.3575534.334.32.73
690.31PremiumGSI161.8585534.354.322.68
700.24PremiumEVVS160.7585534.014.032.44
710.24Very GoodDVVS161.5605533.9742.45
720.3Very GoodHSI163.1565544.294.272.7
730.3PremiumHSI162.9595544.284.242.68
740.3PremiumHSI162.5575544.294.252.67
750.3GoodHSI163.7575544.284.262.72
760.26Very GoodFVVS259.2605544.194.222.49
770.26Very GoodEVVS259.9585544.154.232.51
780.26Very GoodDVVS262.4545544.084.132.56
790.26Very GoodDVVS262.8605544.014.052.53
800.26Very GoodEVVS162.6595544.064.092.55
810.26Very GoodEVVS163.45955444.042.55
820.26Very GoodDVVS162.1605544.034.122.53
830.26IdealEVVS262.9585544.024.062.54
840.38IdealISI261.6565544.654.672.87
850.26GoodEVVS157.9605544.224.252.45
860.24PremiumGVVS162.3595543.953.922.45
870.24PremiumHVVS161.2585544.013.962.44
880.24PremiumHVVS160.8595544.0242.44
890.24PremiumHVVS260.7585544.074.042.46
900.32PremiumISI162.9585544.354.332.73
910.7IdealESI162.55727575.75.723.57
920.86FairESI255.16927576.456.333.52
930.7IdealGVS261.65627575.75.673.5
940.71Very GoodEVS262.45727595.685.733.56
950.78Very GoodGSI263.85627595.815.853.72
960.7GoodEVS257.55827595.855.93.38
970.7GoodFVS159.46227595.715.763.4
980.96FairFSI266.36227596.275.954.07
990.73Very GoodESI161.65927605.775.783.56
1000.8PremiumHSI161.55827605.975.933.66

Showing the first 1000 rows.

%r

printSchema(diamonds)
root
 |-- _c0: integer (nullable = true)
 |-- carat: double (nullable = true)
 |-- cut: string (nullable = true)
 |-- color: string (nullable = true)
 |-- clarity: string (nullable = true)
 |-- depth: double (nullable = true)
 |-- table: double (nullable = true)
 |-- price: integer (nullable = true)
 |-- x: double (nullable = true)
 |-- y: double (nullable = true)
 |-- z: double (nullable = true)
%python

diamonds = spark.read.format('csv').options(header='true', inferSchema='true').load('/databricks-datasets/Rdatasets/data-001/csv/ggplot2/diamonds.csv')

display(diamonds)
10.23IdealESI261.5553263.953.982.43
20.21PremiumESI159.8613263.893.842.31
30.23GoodEVS156.9653274.054.072.31
40.29PremiumIVS262.4583344.24.232.63
50.31GoodJSI263.3583354.344.352.75
60.24Very GoodJVVS262.8573363.943.962.48
70.24Very GoodIVVS162.3573363.953.982.47
80.26Very GoodHSI161.9553374.074.112.53
90.22FairEVS265.1613373.873.782.49
100.23Very GoodHVS159.46133844.052.39
110.3GoodJSI164553394.254.282.73
120.23IdealJVS162.8563403.933.92.46
130.22PremiumFSI160.4613423.883.842.33
140.31IdealJSI262.2543444.354.372.71
150.2PremiumESI260.2623453.793.752.27
160.32PremiumEI160.9583454.384.422.68
170.3IdealISI262543484.314.342.68
180.3GoodJSI163.4543514.234.292.7
190.3GoodJSI163.8563514.234.262.71
200.3Very GoodJSI162.7593514.214.272.66
210.3GoodISI263.3563514.264.32.71
220.23Very GoodEVS263.8553523.853.922.48
230.23Very GoodHVS161573533.943.962.41
240.31Very GoodJSI159.4623534.394.432.62
250.31Very GoodJSI158.1623534.444.472.59
260.23Very GoodGVVS260.4583543.974.012.41
270.24PremiumIVS162.5573553.973.942.47
280.3Very GoodJVS262.2573574.284.32.67
290.23Very GoodDVS260.5613573.963.972.4
300.23Very GoodFVS160.9573573.963.992.42
310.23Very GoodFVS1605740244.032.41
320.23Very GoodFVS159.8574024.044.062.42
330.23Very GoodEVS160.7594023.974.012.42
340.23Very GoodEVS159.5584024.014.062.4
350.23Very GoodDVS161.9584023.923.962.44
360.23GoodFVS158.2594024.064.082.37
370.23GoodEVS164.1594023.833.852.46
380.31GoodHSI164544024.294.312.75
390.26Very GoodDVS260.8594034.134.162.52
400.33IdealISI261.8554034.494.512.78
410.33IdealISI261.2564034.494.52.75
420.33IdealJSI161.1564034.494.552.76
430.26GoodDVS265.2564033.994.022.61
440.26GoodDVS158.4634034.194.242.46
450.32GoodHSI263.1564034.344.372.75
460.29PremiumFSI162.4584034.244.262.65
470.32Very GoodHSI261.8554034.354.422.71
480.32GoodHSI263.8564034.364.382.79
490.25Very GoodEVS263.36040444.032.54
500.29Very GoodHSI260.7604044.334.372.64
510.24Very GoodFSI160.9614044.024.032.45
520.23IdealGVS161.9544043.933.952.44
530.32IdealISI160.9554044.454.482.72
540.22PremiumEVS261.6584043.933.892.41
550.22PremiumDVS259.3624043.913.882.31
560.3IdealISI261594054.34.332.63
570.3PremiumJSI259.3614054.434.382.61
580.3Very GoodISI162.6574054.254.282.67
590.3Very GoodISI163574054.284.322.71
600.3GoodISI163.2554054.254.292.7
610.35IdealIVS160.9575524.544.592.78
620.3PremiumDSI162.6595524.234.272.66
630.3IdealDSI162.5575524.294.322.69
640.3IdealDSI162.1565524.34.332.68
650.42PremiumISI261.5595524.784.842.96
660.28IdealGVVS261.4565534.194.222.58
670.32IdealIVVS16255.35534.394.422.73
680.31Very GoodGSI163.3575534.334.32.73
690.31PremiumGSI161.8585534.354.322.68
700.24PremiumEVVS160.7585534.014.032.44
710.24Very GoodDVVS161.5605533.9742.45
720.3Very GoodHSI163.1565544.294.272.7
730.3PremiumHSI162.9595544.284.242.68
740.3PremiumHSI162.5575544.294.252.67
750.3GoodHSI163.7575544.284.262.72
760.26Very GoodFVVS259.2605544.194.222.49
770.26Very GoodEVVS259.9585544.154.232.51
780.26Very GoodDVVS262.4545544.084.132.56
790.26Very GoodDVVS262.8605544.014.052.53
800.26Very GoodEVVS162.6595544.064.092.55
810.26Very GoodEVVS163.45955444.042.55
820.26Very GoodDVVS162.1605544.034.122.53
830.26IdealEVVS262.9585544.024.062.54
840.38IdealISI261.6565544.654.672.87
850.26GoodEVVS157.9605544.224.252.45
860.24PremiumGVVS162.3595543.953.922.45
870.24PremiumHVVS161.2585544.013.962.44
880.24PremiumHVVS160.8595544.0242.44
890.24PremiumHVVS260.7585544.074.042.46
900.32PremiumISI162.9585544.354.332.73
910.7IdealESI162.55727575.75.723.57
920.86FairESI255.16927576.456.333.52
930.7IdealGVS261.65627575.75.673.5
940.71Very GoodEVS262.45727595.685.733.56
950.78Very GoodGSI263.85627595.815.853.72
960.7GoodEVS257.55827595.855.93.38
970.7GoodFVS159.46227595.715.763.4
980.96FairFSI266.36227596.275.954.07
990.73Very GoodESI161.65927605.775.783.56
1000.8PremiumHSI161.55827605.975.933.66

Showing the first 1000 rows.

%python

diamonds.printSchema()
root |-- _c0: integer (nullable = true) |-- carat: double (nullable = true) |-- cut: string (nullable = true) |-- color: string (nullable = true) |-- clarity: string (nullable = true) |-- depth: double (nullable = true) |-- table: double (nullable = true) |-- price: integer (nullable = true) |-- x: double (nullable = true) |-- y: double (nullable = true) |-- z: double (nullable = true)
%sql
-- mode "FAILFAST" will abort file parsing with a RuntimeException if any malformed lines are encountered
CREATE TEMPORARY VIEW diamonds
USING CSV
OPTIONS (path "/databricks-datasets/Rdatasets/data-001/csv/ggplot2/diamonds.csv", header "true", mode "FAILFAST")
OK
%sql SELECT * FROM diamonds
10.23IdealESI261.5553263.953.982.43
20.21PremiumESI159.8613263.893.842.31
30.23GoodEVS156.9653274.054.072.31
40.29PremiumIVS262.4583344.24.232.63
50.31GoodJSI263.3583354.344.352.75
60.24Very GoodJVVS262.8573363.943.962.48
70.24Very GoodIVVS162.3573363.953.982.47
80.26Very GoodHSI161.9553374.074.112.53
90.22FairEVS265.1613373.873.782.49
100.23Very GoodHVS159.46133844.052.39
110.3GoodJSI164553394.254.282.73
120.23IdealJVS162.8563403.933.92.46
130.22PremiumFSI160.4613423.883.842.33
140.31IdealJSI262.2543444.354.372.71
150.2PremiumESI260.2623453.793.752.27
160.32PremiumEI160.9583454.384.422.68
170.3IdealISI262543484.314.342.68
180.3GoodJSI163.4543514.234.292.7
190.3GoodJSI163.8563514.234.262.71
200.3Very GoodJSI162.7593514.214.272.66
210.3GoodISI263.3563514.264.32.71
220.23Very GoodEVS263.8553523.853.922.48
230.23Very GoodHVS161573533.943.962.41
240.31Very GoodJSI159.4623534.394.432.62
250.31Very GoodJSI158.1623534.444.472.59
260.23Very GoodGVVS260.4583543.974.012.41
270.24PremiumIVS162.5573553.973.942.47
280.3Very GoodJVS262.2573574.284.32.67
290.23Very GoodDVS260.5613573.963.972.4
300.23Very GoodFVS160.9573573.963.992.42
310.23Very GoodFVS1605740244.032.41
320.23Very GoodFVS159.8574024.044.062.42
330.23Very GoodEVS160.7594023.974.012.42
340.23Very GoodEVS159.5584024.014.062.4
350.23Very GoodDVS161.9584023.923.962.44
360.23GoodFVS158.2594024.064.082.37
370.23GoodEVS164.1594023.833.852.46
380.31GoodHSI164544024.294.312.75
390.26Very GoodDVS260.8594034.134.162.52
400.33IdealISI261.8554034.494.512.78
410.33IdealISI261.2564034.494.52.75
420.33IdealJSI161.1564034.494.552.76
430.26GoodDVS265.2564033.994.022.61
440.26GoodDVS158.4634034.194.242.46
450.32GoodHSI263.1564034.344.372.75
460.29PremiumFSI162.4584034.244.262.65
470.32Very GoodHSI261.8554034.354.422.71
480.32GoodHSI263.8564034.364.382.79
490.25Very GoodEVS263.36040444.032.54
500.29Very GoodHSI260.7604044.334.372.64
510.24Very GoodFSI160.9614044.024.032.45
520.23IdealGVS161.9544043.933.952.44
530.32IdealISI160.9554044.454.482.72
540.22PremiumEVS261.6584043.933.892.41
550.22PremiumDVS259.3624043.913.882.31
560.3IdealISI261594054.34.332.63
570.3PremiumJSI259.3614054.434.382.61
580.3Very GoodISI162.6574054.254.282.67
590.3Very GoodISI163574054.284.322.71
600.3GoodISI163.2554054.254.292.7
610.35IdealIVS160.9575524.544.592.78
620.3PremiumDSI162.6595524.234.272.66
630.3IdealDSI162.5575524.294.322.69
640.3IdealDSI162.1565524.34.332.68
650.42PremiumISI261.5595524.784.842.96
660.28IdealGVVS261.4565534.194.222.58
670.32IdealIVVS16255.35534.394.422.73
680.31Very GoodGSI163.3575534.334.32.73
690.31PremiumGSI161.8585534.354.322.68
700.24PremiumEVVS160.7585534.014.032.44
710.24Very GoodDVVS161.5605533.9742.45
720.3Very GoodHSI163.1565544.294.272.7
730.3PremiumHSI162.9595544.284.242.68
740.3PremiumHSI162.5575544.294.252.67
750.3GoodHSI163.7575544.284.262.72
760.26Very GoodFVVS259.2605544.194.222.49
770.26Very GoodEVVS259.9585544.154.232.51
780.26Very GoodDVVS262.4545544.084.132.56
790.26Very GoodDVVS262.8605544.014.052.53
800.26Very GoodEVVS162.6595544.064.092.55
810.26Very GoodEVVS163.45955444.042.55
820.26Very GoodDVVS162.1605544.034.122.53
830.26IdealEVVS262.9585544.024.062.54
840.38IdealISI261.6565544.654.672.87
850.26GoodEVVS157.9605544.224.252.45
860.24PremiumGVVS162.3595543.953.922.45
870.24PremiumHVVS161.2585544.013.962.44
880.24PremiumHVVS160.8595544.0242.44
890.24PremiumHVVS260.7585544.074.042.46
900.32PremiumISI162.9585544.354.332.73
910.7IdealESI162.55727575.75.723.57
920.86FairESI255.16927576.456.333.52
930.7IdealGVS261.65627575.75.673.5
940.71Very GoodEVS262.45727595.685.733.56
950.78Very GoodGSI263.85627595.815.853.72
960.7GoodEVS257.55827595.855.93.38
970.7GoodFVS159.46227595.715.763.4
980.96FairFSI266.36227596.275.954.07
990.73Very GoodESI161.65927605.775.783.56
1000.8PremiumHSI161.55827605.975.933.66

Showing the first 1000 rows.