非纯种程序猿 Thinking all about tech & life

<property>
  <name>dfs.block.replicator.classname</name>
  <value>org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid</value>
</property>

<property>
  <name>raid.codecs.json</name>
  <value>
    [   
      {   
        "id"            : "xor",
        "parity_dir"    : "/raid",
        "stripe_length" : 10, 
        "parity_length" : 1,
        "priority"      : 100,
        "erasure_code"  : "org.apache.hadoop.raid.XorCode",
        "description"   : "XOR code"
      },
      {   
        "id"            : "rs",
        "parity_dir"    : "/raidrs",
        "stripe_length" : 10,
        "parity_length" : 4,
        "priority"      : 300,
        "erasure_code"  : "org.apache.hadoop.raid.ReedSolomonCode",
        "description"   : "ReedSolomonCode code",
      },
    ]   
  </value>
  <description>JSon string that contains all Raid codecs</description>
</property>

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

  <property>
    <name>dfs.data.dir</name>
    <value>/home/bo.jiangb/cluster-data/dfs/data</value>
  </property>
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/home/bo.jiangb/cluster-data</value>
  </property>
  <property>
    <name>dfs.name.dir</name>
    <value>/home/bo.jiangb/cluster-data/dfs/name</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>3</value>
  </property>
  <property>
    <name>dfs.datanode.address</name>
    <value>0.0.0.0:55510</value>
  </property>
  <property>
    <name>dfs.datanode.ipc.address</name>
    <value>0.0.0.0:55520</value>
  </property>
  <property>
    <name>dfs.datanode.http.address</name>
    <value>0.0.0.0:55575</value>
  </property>
  <property>
    <name>dfs.block.replicator.classname</name>
    <value>org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid</value>
  </property>

  <property>
    <name>raid.codecs.json</name>
    <value>
      [
      {
      "id"            : "xor",
      "parity_dir"    : "/raid",
      "stripe_length" : 10,
      "parity_length" : 1,
      "priority"      : 100,
      "erasure_code"  : "org.apache.hadoop.raid.XorCode",
      "description"   : "XOR code"
      },
      {
      "id"            : "rs",
      "parity_dir"    : "/raidrs",
      "stripe_length" : 10,
      "parity_length" : 4,
      "priority"      : 300,
      "erasure_code"  : "org.apache.hadoop.raid.ReedSolomonCode",
      "description"   : "ReedSolomonCode code",
      },
      ]
    </value>
 </property>
 <property>
    <name>mapred.raid.http.address</name>
    <value>hrn:61000</value>
  </property>
  <property>
    <name>raid.server.address</name>
    <value>hrn:60011</value>
  </property>
  <property>
    <name>raid.configmanager.class</name>
    <value>org.apache.hadoop.raid.FileConfigManager</value>
  </property>
  <property>
    <name>raid.config.file</name>
    <value>/home/bo.jiangb/hadoop-0.20/conf/raid-policy.xml</value>
  </property>
</configuration>

[bo.jiangb@dw61.kgb.sqa.cm4 hadoop-0.20]$ ./bin/hadoop raidshell
Usage: java RaidShell
           [-showConfig ]
           [-help [cmd]]
           [-recover srcPath1 corruptOffset]
           [-recoverBlocks path1 path2...]
           [-raidFile <path-to-file> <path-to-raidDir> <XOR|RS>
           [-fsck [path [-threads numthreads] [-count]]]
           [-usefulHar <XOR|RS> [path-to-raid-har]]
           [-checkFile path]
           -purgeParity path <XOR|RS>
           [-checkParity path]

Generic options supported are
-conf <configuration file>     specify an application configuration file
-D <property=value>            use value for given property
-fs <local|namenode:port>      specify a namenode
-jt <local|jobtracker:port>    specify a job tracker
-files <comma separated list of files>    specify comma separated files to be copied to the map reduce cluster
-libjars <comma separated list of jars>    specify comma separated jar files to include in the classpath.
-archives <comma separated list of archives>    specify comma separated archives to be unarchived on the compute machines.

The general command line syntax is
bin/hadoop command [genericOptions] [commandOptions]

./bin/hadoop raidshell -raidFile <path-to-file> <path-to-raidDir> <XOR|RS>

./bin/hadoop raidshell -raidFile /user/bo.jiangb/bigfile2 /back/raidrs XOR

./bin/hadoop raidshell -raidFile /user/bo.jiangb /back/raidrs RS

./bin/hadoop raidshell -fsck [path [-threads numthreads] [-count]] 用例：

[bo.jiangb@drn hadoop-0.20]$ ./bin/hadoop raidshell -fsck /
Running RAID FSCK with 16 threads on /
Querying NameNode for list of corrupt files under /
Processing 0 possibly corrupt files using 16 threads

./bin/hadoop raidshell -recover srcPath1 corruptOffset 用例：

[bo.jiangb@drn hadoop-0.20]$ ./bin/hadoop raidshell -recover /user/bo.jiangb/bigfile2 0
13/06/05 17:38:12 INFO hadoop.RaidShell: RaidShell connecting to dw61/10.232.98.61:60011
13/06/05 17:38:13 INFO hadoop.RaidShell: RaidShell recoverFile for /user/bo.jiangb/bigfile2 corruptOffset 0
13/06/05 17:38:39 INFO hadoop.RaidShell: Raidshell created recovery file /tmp/recovered.1370425093021